diff options
Diffstat (limited to 'llvm/lib/Target/Hexagon')
58 files changed, 3728 insertions, 1352 deletions
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index 1e7862c36ea0..b6763fd9aef0 100644 --- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -641,7 +641,7 @@ bool HexagonAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, return true; return finishBundle(IDLoc, Out); } - MCInst *SubInst = new (getParser().getContext()) MCInst; + MCInst *SubInst = getParser().getContext().createMCInst(); if (matchOneInstruction(*SubInst, IDLoc, Operands, ErrorInfo, MatchingInlineAsm)) { if (InBrackets) @@ -945,7 +945,7 @@ bool HexagonAsmParser::isLabel(AsmToken &Token) { StringRef Raw(String.data(), Third.getString().data() - String.data() + Third.getString().size()); std::string Collapsed = std::string(Raw); - Collapsed.erase(llvm::remove_if(Collapsed, isSpace), Collapsed.end()); + llvm::erase_if(Collapsed, isSpace); StringRef Whole = Collapsed; std::pair<StringRef, StringRef> DotSplit = Whole.split('.'); if (!matchRegister(DotSplit.first.lower())) @@ -997,7 +997,7 @@ OperandMatchResultTy HexagonAsmParser::tryParseRegister(unsigned &RegNo, NeededWorkaround = NeededWorkaround || (Again && !(Contigious && Type)); } std::string Collapsed = std::string(RawString); - Collapsed.erase(llvm::remove_if(Collapsed, isSpace), Collapsed.end()); + llvm::erase_if(Collapsed, isSpace); StringRef FullString = Collapsed; std::pair<StringRef, StringRef> DotSplit = FullString.split('.'); unsigned DotReg = matchRegister(DotSplit.first.lower()); diff --git a/llvm/lib/Target/Hexagon/BitTracker.cpp b/llvm/lib/Target/Hexagon/BitTracker.cpp index 7ef23ef35a74..8bced3cec082 100644 --- a/llvm/lib/Target/Hexagon/BitTracker.cpp +++ b/llvm/lib/Target/Hexagon/BitTracker.cpp @@ -198,10 +198,10 @@ BitTracker::~BitTracker() { // the actual bits of the "self" register. // While this cannot happen in the current implementation, I'm not sure // if this should be ruled out in the future. -bool BT::RegisterCell::meet(const RegisterCell &RC, unsigned SelfR) { +bool BT::RegisterCell::meet(const RegisterCell &RC, Register SelfR) { // An example when "meet" can be invoked with SelfR == 0 is a phi node // with a physical register as an operand. - assert(SelfR == 0 || Register::isVirtualRegister(SelfR)); + assert(SelfR == 0 || SelfR.isVirtual()); bool Changed = false; for (uint16_t i = 0, n = Bits.size(); i < n; ++i) { const BitValue &RCV = RC[i]; @@ -335,13 +335,13 @@ uint16_t BT::MachineEvaluator::getRegBitWidth(const RegisterRef &RR) const { // 1. find a physical register PhysR from the same class as RR.Reg, // 2. find a physical register PhysS that corresponds to PhysR:RR.Sub, // 3. find a register class that contains PhysS. - if (Register::isVirtualRegister(RR.Reg)) { + if (RR.Reg.isVirtual()) { const auto &VC = composeWithSubRegIndex(*MRI.getRegClass(RR.Reg), RR.Sub); return TRI.getRegSizeInBits(VC); } - assert(Register::isPhysicalRegister(RR.Reg)); - Register PhysR = - (RR.Sub == 0) ? Register(RR.Reg) : TRI.getSubReg(RR.Reg, RR.Sub); + assert(RR.Reg.isPhysical()); + MCRegister PhysR = + (RR.Sub == 0) ? RR.Reg.asMCReg() : TRI.getSubReg(RR.Reg, RR.Sub); return getPhysRegBitWidth(PhysR); } @@ -351,10 +351,10 @@ BT::RegisterCell BT::MachineEvaluator::getCell(const RegisterRef &RR, // Physical registers are assumed to be present in the map with an unknown // value. Don't actually insert anything in the map, just return the cell. - if (Register::isPhysicalRegister(RR.Reg)) + if (RR.Reg.isPhysical()) return RegisterCell::self(0, BW); - assert(Register::isVirtualRegister(RR.Reg)); + assert(RR.Reg.isVirtual()); // For virtual registers that belong to a class that is not tracked, // generate an "unknown" value as well. const TargetRegisterClass *C = MRI.getRegClass(RR.Reg); @@ -377,7 +377,7 @@ void BT::MachineEvaluator::putCell(const RegisterRef &RR, RegisterCell RC, // While updating the cell map can be done in a meaningful way for // a part of a register, it makes little sense to implement it as the // SSA representation would never contain such "partial definitions". - if (!Register::isVirtualRegister(RR.Reg)) + if (!RR.Reg.isVirtual()) return; assert(RR.Sub == 0 && "Unexpected sub-register in definition"); // Eliminate all ref-to-reg-0 bit values: replace them with "self". @@ -704,15 +704,14 @@ BT::RegisterCell BT::MachineEvaluator::eINS(const RegisterCell &A1, return Res; } -BT::BitMask BT::MachineEvaluator::mask(unsigned Reg, unsigned Sub) const { +BT::BitMask BT::MachineEvaluator::mask(Register Reg, unsigned Sub) const { assert(Sub == 0 && "Generic BitTracker::mask called for Sub != 0"); uint16_t W = getRegBitWidth(Reg); assert(W > 0 && "Cannot generate mask for empty register"); return BitMask(0, W-1); } -uint16_t BT::MachineEvaluator::getPhysRegBitWidth(unsigned Reg) const { - assert(Register::isPhysicalRegister(Reg)); +uint16_t BT::MachineEvaluator::getPhysRegBitWidth(MCRegister Reg) const { const TargetRegisterClass &PC = *TRI.getMinimalPhysRegClass(Reg); return TRI.getRegSizeInBits(PC); } @@ -875,7 +874,7 @@ void BT::visitNonBranch(const MachineInstr &MI) { continue; RegisterRef RD(MO); assert(RD.Sub == 0 && "Unexpected sub-register in definition"); - if (!Register::isVirtualRegister(RD.Reg)) + if (!RD.Reg.isVirtual()) continue; bool Changed = false; @@ -980,7 +979,7 @@ void BT::visitBranchesFrom(const MachineInstr &BI) { FlowQ.push(CFGEdge(ThisN, TB->getNumber())); } -void BT::visitUsesOf(unsigned Reg) { +void BT::visitUsesOf(Register Reg) { if (Trace) dbgs() << "queuing uses of modified reg " << printReg(Reg, &ME.TRI) << " cell: " << ME.getCell(Reg, Map) << '\n'; diff --git a/llvm/lib/Target/Hexagon/BitTracker.h b/llvm/lib/Target/Hexagon/BitTracker.h index efb21805b801..08c0359a4b7f 100644 --- a/llvm/lib/Target/Hexagon/BitTracker.h +++ b/llvm/lib/Target/Hexagon/BitTracker.h @@ -62,7 +62,7 @@ private: void visitPHI(const MachineInstr &PI); void visitNonBranch(const MachineInstr &MI); void visitBranchesFrom(const MachineInstr &BI); - void visitUsesOf(unsigned Reg); + void visitUsesOf(Register Reg); using CFGEdge = std::pair<int, int>; using EdgeSetType = std::set<CFGEdge>; @@ -131,19 +131,20 @@ struct BitTracker::BitRef { return Reg == BR.Reg && (Reg == 0 || Pos == BR.Pos); } - unsigned Reg; + Register Reg; uint16_t Pos; }; // Abstraction of a register reference in MachineOperand. It contains the // register number and the subregister index. +// FIXME: Consolidate duplicate definitions of RegisterRef struct BitTracker::RegisterRef { - RegisterRef(unsigned R = 0, unsigned S = 0) - : Reg(R), Sub(S) {} + RegisterRef(Register R = 0, unsigned S = 0) : Reg(R), Sub(S) {} RegisterRef(const MachineOperand &MO) : Reg(MO.getReg()), Sub(MO.getSubReg()) {} - unsigned Reg, Sub; + Register Reg; + unsigned Sub; }; // Value that a single bit can take. This is outside of the context of @@ -312,7 +313,7 @@ struct BitTracker::RegisterCell { return Bits[BitN]; } - bool meet(const RegisterCell &RC, unsigned SelfR); + bool meet(const RegisterCell &RC, Register SelfR); RegisterCell &insert(const RegisterCell &RC, const BitMask &M); RegisterCell extract(const BitMask &M) const; // Returns a new cell. RegisterCell &rol(uint16_t Sh); // Rotate left. @@ -461,7 +462,7 @@ struct BitTracker::MachineEvaluator { // Sub == 0, in this case, the function should return a mask that spans // the entire register Reg (which is what the default implementation // does). - virtual BitMask mask(unsigned Reg, unsigned Sub) const; + virtual BitMask mask(Register Reg, unsigned Sub) const; // Indicate whether a given register class should be tracked. virtual bool track(const TargetRegisterClass *RC) const { return true; } // Evaluate a non-branching machine instruction, given the cell map with @@ -484,7 +485,7 @@ struct BitTracker::MachineEvaluator { llvm_unreachable("Unimplemented composeWithSubRegIndex"); } // Return the size in bits of the physical register Reg. - virtual uint16_t getPhysRegBitWidth(unsigned Reg) const; + virtual uint16_t getPhysRegBitWidth(MCRegister Reg) const; const TargetRegisterInfo &TRI; MachineRegisterInfo &MRI; diff --git a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp index f3a87ef20a60..aeaeac65de96 100644 --- a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp +++ b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp @@ -175,7 +175,7 @@ DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size, while (Result == Success && !Complete) { if (Bytes.size() < HEXAGON_INSTR_SIZE) return MCDisassembler::Fail; - MCInst *Inst = new (getContext()) MCInst; + MCInst *Inst = getContext().createMCInst(); Result = getSingleInstruction(*Inst, MI, Bytes, Address, cs, Complete); MI.addOperand(MCOperand::createInst(Inst)); Size += HEXAGON_INSTR_SIZE; @@ -384,8 +384,8 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB, break; } MI.setOpcode(Hexagon::DuplexIClass0 + duplexIClass); - MCInst *MILow = new (getContext()) MCInst; - MCInst *MIHigh = new (getContext()) MCInst; + MCInst *MILow = getContext().createMCInst(); + MCInst *MIHigh = getContext().createMCInst(); auto TmpExtender = CurrentExtender; CurrentExtender = nullptr; // constant extenders in duplex must always be in slot 1 diff --git a/llvm/lib/Target/Hexagon/Hexagon.h b/llvm/lib/Target/Hexagon/Hexagon.h index 58dadf012da5..98e5710d4fc1 100644 --- a/llvm/lib/Target/Hexagon/Hexagon.h +++ b/llvm/lib/Target/Hexagon/Hexagon.h @@ -14,12 +14,9 @@ #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGON_H #define LLVM_LIB_TARGET_HEXAGON_HEXAGON_H -#include "MCTargetDesc/HexagonMCTargetDesc.h" -#include "llvm/CodeGen/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" - namespace llvm { class HexagonTargetMachine; + class ImmutablePass; /// Creates a Hexagon-specific Target Transformation Info pass. ImmutablePass *createHexagonTargetTransformInfoPass(const HexagonTargetMachine *TM); diff --git a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h index 3932def87854..3932def87854 100755..100644 --- a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h +++ b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h diff --git a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp index 49edb0d99492..54aa14849dd9 100644 --- a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp +++ b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp @@ -206,14 +206,14 @@ namespace { uint16_t W); static bool getConst(const BitTracker::RegisterCell &RC, uint16_t B, uint16_t W, uint64_t &U); - static bool replaceReg(unsigned OldR, unsigned NewR, - MachineRegisterInfo &MRI); + static bool replaceReg(Register OldR, Register NewR, + MachineRegisterInfo &MRI); static bool getSubregMask(const BitTracker::RegisterRef &RR, unsigned &Begin, unsigned &Width, MachineRegisterInfo &MRI); - static bool replaceRegWithSub(unsigned OldR, unsigned NewR, - unsigned NewSR, MachineRegisterInfo &MRI); - static bool replaceSubWithSub(unsigned OldR, unsigned OldSR, - unsigned NewR, unsigned NewSR, MachineRegisterInfo &MRI); + static bool replaceRegWithSub(Register OldR, Register NewR, unsigned NewSR, + MachineRegisterInfo &MRI); + static bool replaceSubWithSub(Register OldR, unsigned OldSR, Register NewR, + unsigned NewSR, MachineRegisterInfo &MRI); static bool parseRegSequence(const MachineInstr &I, BitTracker::RegisterRef &SL, BitTracker::RegisterRef &SH, const MachineRegisterInfo &MRI); @@ -292,7 +292,7 @@ void HexagonBitSimplify::getInstrDefs(const MachineInstr &MI, if (!Op.isReg() || !Op.isDef()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; Defs.insert(R); } @@ -304,7 +304,7 @@ void HexagonBitSimplify::getInstrUses(const MachineInstr &MI, if (!Op.isReg() || !Op.isUse()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; Uses.insert(R); } @@ -352,9 +352,9 @@ bool HexagonBitSimplify::getConst(const BitTracker::RegisterCell &RC, return true; } -bool HexagonBitSimplify::replaceReg(unsigned OldR, unsigned NewR, - MachineRegisterInfo &MRI) { - if (!Register::isVirtualRegister(OldR) || !Register::isVirtualRegister(NewR)) +bool HexagonBitSimplify::replaceReg(Register OldR, Register NewR, + MachineRegisterInfo &MRI) { + if (!OldR.isVirtual() || !NewR.isVirtual()) return false; auto Begin = MRI.use_begin(OldR), End = MRI.use_end(); decltype(End) NextI; @@ -365,9 +365,10 @@ bool HexagonBitSimplify::replaceReg(unsigned OldR, unsigned NewR, return Begin != End; } -bool HexagonBitSimplify::replaceRegWithSub(unsigned OldR, unsigned NewR, - unsigned NewSR, MachineRegisterInfo &MRI) { - if (!Register::isVirtualRegister(OldR) || !Register::isVirtualRegister(NewR)) +bool HexagonBitSimplify::replaceRegWithSub(Register OldR, Register NewR, + unsigned NewSR, + MachineRegisterInfo &MRI) { + if (!OldR.isVirtual() || !NewR.isVirtual()) return false; if (hasTiedUse(OldR, MRI, NewSR)) return false; @@ -381,9 +382,10 @@ bool HexagonBitSimplify::replaceRegWithSub(unsigned OldR, unsigned NewR, return Begin != End; } -bool HexagonBitSimplify::replaceSubWithSub(unsigned OldR, unsigned OldSR, - unsigned NewR, unsigned NewSR, MachineRegisterInfo &MRI) { - if (!Register::isVirtualRegister(OldR) || !Register::isVirtualRegister(NewR)) +bool HexagonBitSimplify::replaceSubWithSub(Register OldR, unsigned OldSR, + Register NewR, unsigned NewSR, + MachineRegisterInfo &MRI) { + if (!OldR.isVirtual() || !NewR.isVirtual()) return false; if (OldSR != NewSR && hasTiedUse(OldR, MRI, NewSR)) return false; @@ -894,7 +896,7 @@ bool HexagonBitSimplify::getUsedBits(unsigned Opc, unsigned OpN, // register class. const TargetRegisterClass *HexagonBitSimplify::getFinalVRegClass( const BitTracker::RegisterRef &RR, MachineRegisterInfo &MRI) { - if (!Register::isVirtualRegister(RR.Reg)) + if (!RR.Reg.isVirtual()) return nullptr; auto *RC = MRI.getRegClass(RR.Reg); if (RR.Sub == 0) @@ -925,8 +927,7 @@ const TargetRegisterClass *HexagonBitSimplify::getFinalVRegClass( // with a 32-bit register. bool HexagonBitSimplify::isTransparentCopy(const BitTracker::RegisterRef &RD, const BitTracker::RegisterRef &RS, MachineRegisterInfo &MRI) { - if (!Register::isVirtualRegister(RD.Reg) || - !Register::isVirtualRegister(RS.Reg)) + if (!RD.Reg.isVirtual() || !RS.Reg.isVirtual()) return false; // Return false if one (or both) classes are nullptr. auto *DRC = getFinalVRegClass(RD, MRI); @@ -1017,7 +1018,7 @@ bool DeadCodeElimination::runOnNode(MachineDomTreeNode *N) { if (!Op.isReg() || !Op.isDef()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R) || !isDead(R)) { + if (!R.isVirtual() || !isDead(R)) { AllDead = false; break; } @@ -1219,7 +1220,7 @@ bool RedundantInstrElimination::computeUsedBits(unsigned Reg, BitVector &Bits) { MachineInstr &UseI = *I->getParent(); if (UseI.isPHI() || UseI.isCopy()) { Register DefR = UseI.getOperand(0).getReg(); - if (!Register::isVirtualRegister(DefR)) + if (!DefR.isVirtual()) return false; Pending.push_back(DefR); } else { @@ -1380,8 +1381,9 @@ namespace { static bool isTfrConst(const MachineInstr &MI); private: - unsigned genTfrConst(const TargetRegisterClass *RC, int64_t C, - MachineBasicBlock &B, MachineBasicBlock::iterator At, DebugLoc &DL); + Register genTfrConst(const TargetRegisterClass *RC, int64_t C, + MachineBasicBlock &B, MachineBasicBlock::iterator At, + DebugLoc &DL); const HexagonInstrInfo &HII; MachineRegisterInfo &MRI; @@ -1408,8 +1410,10 @@ bool ConstGeneration::isTfrConst(const MachineInstr &MI) { // Generate a transfer-immediate instruction that is appropriate for the // register class and the actual value being transferred. -unsigned ConstGeneration::genTfrConst(const TargetRegisterClass *RC, int64_t C, - MachineBasicBlock &B, MachineBasicBlock::iterator At, DebugLoc &DL) { +Register ConstGeneration::genTfrConst(const TargetRegisterClass *RC, int64_t C, + MachineBasicBlock &B, + MachineBasicBlock::iterator At, + DebugLoc &DL) { Register Reg = MRI.createVirtualRegister(RC); if (RC == &Hexagon::IntRegsRegClass) { BuildMI(B, At, DL, HII.get(Hexagon::A2_tfrsi), Reg) @@ -1473,8 +1477,8 @@ bool ConstGeneration::processBlock(MachineBasicBlock &B, const RegisterSet&) { HBS::getInstrDefs(*I, Defs); if (Defs.count() != 1) continue; - unsigned DR = Defs.find_first(); - if (!Register::isVirtualRegister(DR)) + Register DR = Defs.find_first(); + if (!DR.isVirtual()) continue; uint64_t U; const BitTracker::RegisterCell &DRC = BT.lookup(DR); @@ -1482,7 +1486,7 @@ bool ConstGeneration::processBlock(MachineBasicBlock &B, const RegisterSet&) { int64_t C = U; DebugLoc DL = I->getDebugLoc(); auto At = I->isPHI() ? B.getFirstNonPHI() : I; - unsigned ImmReg = genTfrConst(MRI.getRegClass(DR), C, B, At, DL); + Register ImmReg = genTfrConst(MRI.getRegClass(DR), C, B, At, DL); if (ImmReg) { HBS::replaceReg(DR, ImmReg, MRI); BT.put(ImmReg, DRC); @@ -1549,7 +1553,7 @@ bool CopyGeneration::findMatch(const BitTracker::RegisterRef &Inp, if (!HBS::getSubregMask(Inp, B, W, MRI)) return false; - for (unsigned R = AVs.find_first(); R; R = AVs.find_next(R)) { + for (Register R = AVs.find_first(); R; R = AVs.find_next(R)) { if (!BT.has(R) || Forbidden[R]) continue; const BitTracker::RegisterCell &RC = BT.lookup(R); @@ -1608,7 +1612,7 @@ bool CopyGeneration::processBlock(MachineBasicBlock &B, DebugLoc DL = I->getDebugLoc(); auto At = I->isPHI() ? B.getFirstNonPHI() : I; - for (unsigned R = Defs.find_first(); R; R = Defs.find_next(R)) { + for (Register R = Defs.find_first(); R; R = Defs.find_next(R)) { BitTracker::RegisterRef MR; auto *FRC = HBS::getFinalVRegClass(R, MRI); @@ -1815,7 +1819,7 @@ bool BitSimplification::matchHalf(unsigned SelfR, if (I == B+16) return false; - unsigned Reg = RC[I].RefI.Reg; + Register Reg = RC[I].RefI.Reg; unsigned P = RC[I].RefI.Pos; // The RefI.Pos will be advanced by I-B. if (P < I-B) return false; @@ -1823,7 +1827,7 @@ bool BitSimplification::matchHalf(unsigned SelfR, if (Reg == 0 || Reg == SelfR) // Don't match "self". return false; - if (!Register::isVirtualRegister(Reg)) + if (!Reg.isVirtual()) return false; if (!BT.has(Reg)) return false; @@ -2363,7 +2367,7 @@ bool BitSimplification::simplifyTstbit(MachineInstr *MI, P = V.RefI.Pos; } if (P != std::numeric_limits<unsigned>::max()) { - unsigned NewR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass); + Register NewR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass); BuildMI(B, At, DL, HII.get(Hexagon::S2_tstbit_i), NewR) .addReg(RR.Reg, 0, RR.Sub) .addImm(P); @@ -3165,8 +3169,8 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) { HBS::getInstrDefs(*I, Defs); if (Defs.count() != 1) continue; - unsigned DefR = Defs.find_first(); - if (!Register::isVirtualRegister(DefR)) + Register DefR = Defs.find_first(); + if (!DefR.isVirtual()) continue; if (!isBitShuffle(&*I, DefR)) continue; diff --git a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp index 1e4030b84bc1..0f6dedeb28c3 100644 --- a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp +++ b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp @@ -86,7 +86,7 @@ HexagonEvaluator::HexagonEvaluator(const HexagonRegisterInfo &tri, } } -BT::BitMask HexagonEvaluator::mask(unsigned Reg, unsigned Sub) const { +BT::BitMask HexagonEvaluator::mask(Register Reg, unsigned Sub) const { if (Sub == 0) return MachineEvaluator::mask(Reg, 0); const TargetRegisterClass &RC = *MRI.getRegClass(Reg); @@ -110,9 +110,7 @@ BT::BitMask HexagonEvaluator::mask(unsigned Reg, unsigned Sub) const { llvm_unreachable("Unexpected register/subregister"); } -uint16_t HexagonEvaluator::getPhysRegBitWidth(unsigned Reg) const { - assert(Register::isPhysicalRegister(Reg)); - +uint16_t HexagonEvaluator::getPhysRegBitWidth(MCRegister Reg) const { using namespace Hexagon; const auto &HST = MF.getSubtarget<HexagonSubtarget>(); if (HST.useHVXOps()) { @@ -1043,7 +1041,7 @@ unsigned HexagonEvaluator::getUniqueDefVReg(const MachineInstr &MI) const { if (!Op.isReg() || !Op.isDef()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; if (DefReg != 0) return 0; diff --git a/llvm/lib/Target/Hexagon/HexagonBitTracker.h b/llvm/lib/Target/Hexagon/HexagonBitTracker.h index 02607d50f686..2d24e859e761 100644 --- a/llvm/lib/Target/Hexagon/HexagonBitTracker.h +++ b/llvm/lib/Target/Hexagon/HexagonBitTracker.h @@ -36,9 +36,9 @@ struct HexagonEvaluator : public BitTracker::MachineEvaluator { bool evaluate(const MachineInstr &BI, const CellMapType &Inputs, BranchTargetList &Targets, bool &FallsThru) const override; - BitTracker::BitMask mask(unsigned Reg, unsigned Sub) const override; + BitTracker::BitMask mask(Register Reg, unsigned Sub) const override; - uint16_t getPhysRegBitWidth(unsigned Reg) const override; + uint16_t getPhysRegBitWidth(MCRegister Reg) const override; const TargetRegisterClass &composeWithSubRegIndex( const TargetRegisterClass &RC, unsigned Idx) const override; diff --git a/llvm/lib/Target/Hexagon/HexagonBlockRanges.cpp b/llvm/lib/Target/Hexagon/HexagonBlockRanges.cpp index d1d1b8ee7d41..56ee3cd60c17 100644 --- a/llvm/lib/Target/Hexagon/HexagonBlockRanges.cpp +++ b/llvm/lib/Target/Hexagon/HexagonBlockRanges.cpp @@ -84,7 +84,7 @@ void HexagonBlockRanges::RangeList::unionize(bool MergeAdjacent) { if (empty()) return; - llvm::sort(begin(), end()); + llvm::sort(*this); iterator Iter = begin(); while (Iter != end()-1) { @@ -275,7 +275,7 @@ HexagonBlockRanges::RegisterSet HexagonBlockRanges::expandToSubRegs( for (; I.isValid(); ++I) SRs.insert({*I, 0}); } else { - assert(Register::isVirtualRegister(R.Reg)); + assert(R.Reg.isVirtual()); auto &RC = *MRI.getRegClass(R.Reg); unsigned PReg = *RC.begin(); MCSubRegIndexIterator I(PReg, &TRI); @@ -482,7 +482,7 @@ HexagonBlockRanges::RegToRangeMap HexagonBlockRanges::computeDeadMap( } } for (auto &P : LiveMap) - if (Register::isVirtualRegister(P.first.Reg)) + if (P.first.Reg.isVirtual()) addDeadRanges(P.first); LLVM_DEBUG(dbgs() << __func__ << ": dead map\n" diff --git a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h index 61115e29a708..5a3b6433fba7 100644 --- a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h +++ b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_HEXAGON_HEXAGONBLOCKRANGES_H #include "llvm/ADT/BitVector.h" +#include "llvm/CodeGen/Register.h" #include <cassert> #include <map> #include <set> @@ -30,8 +31,10 @@ class TargetRegisterInfo; struct HexagonBlockRanges { HexagonBlockRanges(MachineFunction &MF); + // FIXME: Consolidate duplicate definitions of RegisterRef struct RegisterRef { - unsigned Reg, Sub; + llvm::Register Reg; + unsigned Sub; bool operator<(RegisterRef R) const { return Reg < R.Reg || (Reg == R.Reg && Sub < R.Sub); diff --git a/llvm/lib/Target/Hexagon/HexagonCFGOptimizer.cpp b/llvm/lib/Target/Hexagon/HexagonCFGOptimizer.cpp index 11a455ce4347..b456cf139c55 100644 --- a/llvm/lib/Target/Hexagon/HexagonCFGOptimizer.cpp +++ b/llvm/lib/Target/Hexagon/HexagonCFGOptimizer.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "Hexagon.h" +#include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunction.h" diff --git a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp index 6a5192c866cc..11e7d5a17fa9 100644 --- a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp +++ b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp @@ -447,7 +447,7 @@ static void nodes_for_root(GepNode *Root, NodeChildrenMap &NCM, Work.erase(First); NodeChildrenMap::iterator CF = NCM.find(N); if (CF != NCM.end()) { - Work.insert(Work.end(), CF->second.begin(), CF->second.end()); + llvm::append_range(Work, CF->second); Nodes.insert(CF->second.begin(), CF->second.end()); } } @@ -472,10 +472,11 @@ static const NodeSet *node_class(GepNode *N, NodeSymRel &Rel) { // determining equality. The only purpose of the ordering is to eliminate // duplication due to the commutativity of equality/non-equality. static NodePair node_pair(GepNode *N1, GepNode *N2) { - uintptr_t P1 = uintptr_t(N1), P2 = uintptr_t(N2); - if (P1 <= P2) - return std::make_pair(N1, N2); - return std::make_pair(N2, N1); + uintptr_t P1 = reinterpret_cast<uintptr_t>(N1); + uintptr_t P2 = reinterpret_cast<uintptr_t>(N2); + if (P1 <= P2) + return std::make_pair(N1, N2); + return std::make_pair(N2, N1); } static unsigned node_hash(GepNode *N) { @@ -650,8 +651,7 @@ void HexagonCommonGEP::common() { // Node for removal. Erase.insert(*I); } - NodeVect::iterator NewE = remove_if(Nodes, in_set(Erase)); - Nodes.resize(std::distance(Nodes.begin(), NewE)); + erase_if(Nodes, in_set(Erase)); LLVM_DEBUG(dbgs() << "Gep nodes after post-commoning cleanup:\n" << Nodes); } @@ -1145,7 +1145,7 @@ void HexagonCommonGEP::getAllUsersForNode(GepNode *Node, ValueVect &Values, NodeChildrenMap::iterator CF = NCM.find(N); if (CF != NCM.end()) { NodeVect &Cs = CF->second; - Work.insert(Work.end(), Cs.begin(), Cs.end()); + llvm::append_range(Work, Cs); } } } diff --git a/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp b/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp index 05b95d8b7314..a774baaa48e6 100644 --- a/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp +++ b/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp @@ -242,18 +242,15 @@ namespace { return *this; } bool isVReg() const { - return Reg != 0 && !llvm::Register::isStackSlot(Reg) && - llvm::Register::isVirtualRegister(Reg); - } - bool isSlot() const { - return Reg != 0 && llvm::Register::isStackSlot(Reg); + return Reg != 0 && !Reg.isStack() && Reg.isVirtual(); } + bool isSlot() const { return Reg != 0 && Reg.isStack(); } operator MachineOperand() const { if (isVReg()) return MachineOperand::CreateReg(Reg, /*Def*/false, /*Imp*/false, /*Kill*/false, /*Dead*/false, /*Undef*/false, /*EarlyClobber*/false, Sub); - if (llvm::Register::isStackSlot(Reg)) { + if (Reg.isStack()) { int FI = llvm::Register::stackSlot2Index(Reg); return MachineOperand::CreateFI(FI); } @@ -265,7 +262,8 @@ namespace { // For std::map. return Reg < R.Reg || (Reg == R.Reg && Sub < R.Sub); } - unsigned Reg = 0, Sub = 0; + llvm::Register Reg; + unsigned Sub = 0; }; struct ExtExpr { diff --git a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp index 77578378b058..4a2b0600f42b 100644 --- a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp +++ b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp @@ -83,7 +83,8 @@ namespace { // FIXME: Use TargetInstrInfo::RegSubRegPair. Also duplicated in // HexagonGenPredicate struct RegisterSubReg { - unsigned Reg, SubReg; + Register Reg; + unsigned SubReg; explicit RegisterSubReg(unsigned R, unsigned SR = 0) : Reg(R), SubReg(SR) {} explicit RegisterSubReg(const MachineOperand &MO) @@ -216,16 +217,16 @@ namespace { void clear() { Map.clear(); } - bool has(unsigned R) const { + bool has(Register R) const { // All non-virtual registers are considered "bottom". - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) return true; MapType::const_iterator F = Map.find(R); return F != Map.end(); } - const LatticeCell &get(unsigned R) const { - if (!Register::isVirtualRegister(R)) + const LatticeCell &get(Register R) const { + if (!R.isVirtual()) return Bottom; MapType::const_iterator F = Map.find(R); if (F != Map.end()) @@ -234,14 +235,12 @@ namespace { } // Invalidates any const references. - void update(unsigned R, const LatticeCell &L) { - Map[R] = L; - } + void update(Register R, const LatticeCell &L) { Map[R] = L; } void print(raw_ostream &os, const TargetRegisterInfo &TRI) const; private: - using MapType = std::map<unsigned, LatticeCell>; + using MapType = std::map<Register, LatticeCell>; MapType Map; // To avoid creating "top" entries, return a const reference to @@ -633,7 +632,7 @@ void MachineConstPropagator::visitPHI(const MachineInstr &PN) { const MachineOperand &MD = PN.getOperand(0); RegisterSubReg DefR(MD); - assert(Register::isVirtualRegister(DefR.Reg)); + assert(DefR.Reg.isVirtual()); bool Changed = false; @@ -662,7 +661,7 @@ Bottomize: RegisterSubReg UseR(SO); // If the input is not a virtual register, we don't really know what // value it holds. - if (!Register::isVirtualRegister(UseR.Reg)) + if (!UseR.Reg.isVirtual()) goto Bottomize; // If there is no cell for an input register, it means top. if (!Cells.has(UseR.Reg)) @@ -704,7 +703,7 @@ void MachineConstPropagator::visitNonBranch(const MachineInstr &MI) { continue; RegisterSubReg DefR(MO); // Only track virtual registers. - if (!Register::isVirtualRegister(DefR.Reg)) + if (!DefR.Reg.isVirtual()) continue; bool Changed = false; // If the evaluation failed, set cells for all output registers to bottom. @@ -1086,7 +1085,7 @@ bool MachineConstPropagator::run(MachineFunction &MF) { bool MachineConstEvaluator::getCell(const RegisterSubReg &R, const CellMap &Inputs, LatticeCell &RC) { - if (!Register::isVirtualRegister(R.Reg)) + if (!R.Reg.isVirtual()) return false; const LatticeCell &L = Inputs.get(R.Reg); if (!R.SubReg) { @@ -1884,7 +1883,7 @@ namespace { bool evaluateHexVector2(const MachineInstr &MI, const CellMap &Inputs, CellMap &Outputs); - void replaceAllRegUsesWith(unsigned FromReg, unsigned ToReg); + void replaceAllRegUsesWith(Register FromReg, Register ToReg); bool rewriteHexBranch(MachineInstr &BrI, const CellMap &Inputs); bool rewriteHexConstDefs(MachineInstr &MI, const CellMap &Inputs, bool &AllDefs); @@ -1942,7 +1941,7 @@ bool HexagonConstEvaluator::evaluate(const MachineInstr &MI, unsigned Opc = MI.getOpcode(); RegisterSubReg DefR(MD); assert(!DefR.SubReg); - if (!Register::isVirtualRegister(DefR.Reg)) + if (!DefR.Reg.isVirtual()) return false; if (MI.isCopy()) { @@ -2809,7 +2808,7 @@ bool HexagonConstEvaluator::rewriteHexConstDefs(MachineInstr &MI, if (!MO.isReg() || !MO.isUse() || MO.isImplicit()) continue; RegisterSubReg R(MO); - if (!Register::isVirtualRegister(R.Reg)) + if (!R.Reg.isVirtual()) continue; HasUse = true; // PHIs can legitimately have "top" cells after propagation. @@ -2851,7 +2850,7 @@ bool HexagonConstEvaluator::rewriteHexConstDefs(MachineInstr &MI, if (!MO.isReg() || !MO.isDef()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; assert(!MO.getSubReg()); assert(Inputs.has(R)); @@ -3130,10 +3129,10 @@ bool HexagonConstEvaluator::rewriteHexConstUses(MachineInstr &MI, return Changed; } -void HexagonConstEvaluator::replaceAllRegUsesWith(unsigned FromReg, - unsigned ToReg) { - assert(Register::isVirtualRegister(FromReg)); - assert(Register::isVirtualRegister(ToReg)); +void HexagonConstEvaluator::replaceAllRegUsesWith(Register FromReg, + Register ToReg) { + assert(FromReg.isVirtual()); + assert(ToReg.isVirtual()); for (auto I = MRI->use_begin(FromReg), E = MRI->use_end(); I != E;) { MachineOperand &O = *I; ++I; diff --git a/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp b/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp index 587527d8c32c..23d0cc829e52 100644 --- a/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonCopyToCombine.cpp @@ -10,6 +10,7 @@ // to move them together. If we can move them next to each other we do so and // replace them with a combine instruction. //===----------------------------------------------------------------------===// + #include "HexagonInstrInfo.h" #include "HexagonSubtarget.h" #include "llvm/ADT/DenseMap.h" @@ -26,6 +27,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; diff --git a/llvm/lib/Target/Hexagon/HexagonEarlyIfConv.cpp b/llvm/lib/Target/Hexagon/HexagonEarlyIfConv.cpp index a431af17e6d0..d36ffc3da641 100644 --- a/llvm/lib/Target/Hexagon/HexagonEarlyIfConv.cpp +++ b/llvm/lib/Target/Hexagon/HexagonEarlyIfConv.cpp @@ -386,7 +386,7 @@ bool HexagonEarlyIfConversion::isValidCandidate(const MachineBasicBlock *B) if (!MO.isReg() || !MO.isDef()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; if (!isPredicate(R)) continue; @@ -403,7 +403,7 @@ bool HexagonEarlyIfConversion::usesUndefVReg(const MachineInstr *MI) const { if (!MO.isReg() || !MO.isUse()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; const MachineInstr *DefI = MRI->getVRegDef(R); // "Undefined" virtual registers are actually defined via IMPLICIT_DEF. @@ -493,7 +493,7 @@ unsigned HexagonEarlyIfConversion::countPredicateDefs( if (!MO.isReg() || !MO.isDef()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; if (isPredicate(R)) PredDefs++; diff --git a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp index c1d0599830cc..fcc880463925 100644 --- a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp +++ b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp @@ -174,6 +174,7 @@ namespace { unsigned CoaCounter = 0; unsigned TfrCounter = 0; + // FIXME: Consolidate duplicate definitions of RegisterRef struct RegisterRef { RegisterRef(const MachineOperand &Op) : Reg(Op.getReg()), Sub(Op.getSubReg()) {} @@ -187,7 +188,8 @@ namespace { return Reg < RR.Reg || (Reg == RR.Reg && Sub < RR.Sub); } - unsigned Reg, Sub; + Register Reg; + unsigned Sub; }; using ReferenceMap = DenseMap<unsigned, unsigned>; @@ -196,25 +198,25 @@ namespace { unsigned getMaskForSub(unsigned Sub); bool isCondset(const MachineInstr &MI); - LaneBitmask getLaneMask(unsigned Reg, unsigned Sub); + LaneBitmask getLaneMask(Register Reg, unsigned Sub); void addRefToMap(RegisterRef RR, ReferenceMap &Map, unsigned Exec); bool isRefInMap(RegisterRef, ReferenceMap &Map, unsigned Exec); - void updateDeadsInRange(unsigned Reg, LaneBitmask LM, LiveRange &Range); - void updateKillFlags(unsigned Reg); - void updateDeadFlags(unsigned Reg); - void recalculateLiveInterval(unsigned Reg); + void updateDeadsInRange(Register Reg, LaneBitmask LM, LiveRange &Range); + void updateKillFlags(Register Reg); + void updateDeadFlags(Register Reg); + void recalculateLiveInterval(Register Reg); void removeInstr(MachineInstr &MI); - void updateLiveness(std::set<unsigned> &RegSet, bool Recalc, - bool UpdateKills, bool UpdateDeads); + void updateLiveness(std::set<Register> &RegSet, bool Recalc, + bool UpdateKills, bool UpdateDeads); unsigned getCondTfrOpcode(const MachineOperand &SO, bool Cond); MachineInstr *genCondTfrFor(MachineOperand &SrcOp, MachineBasicBlock::iterator At, unsigned DstR, unsigned DstSR, const MachineOperand &PredOp, bool PredSense, bool ReadUndef, bool ImpUse); - bool split(MachineInstr &MI, std::set<unsigned> &UpdRegs); + bool split(MachineInstr &MI, std::set<Register> &UpdRegs); bool isPredicable(MachineInstr *MI); MachineInstr *getReachingDefForPred(RegisterRef RD, @@ -224,19 +226,18 @@ namespace { void predicateAt(const MachineOperand &DefOp, MachineInstr &MI, MachineBasicBlock::iterator Where, const MachineOperand &PredOp, bool Cond, - std::set<unsigned> &UpdRegs); + std::set<Register> &UpdRegs); void renameInRange(RegisterRef RO, RegisterRef RN, unsigned PredR, bool Cond, MachineBasicBlock::iterator First, MachineBasicBlock::iterator Last); - bool predicate(MachineInstr &TfrI, bool Cond, std::set<unsigned> &UpdRegs); - bool predicateInBlock(MachineBasicBlock &B, - std::set<unsigned> &UpdRegs); + bool predicate(MachineInstr &TfrI, bool Cond, std::set<Register> &UpdRegs); + bool predicateInBlock(MachineBasicBlock &B, std::set<Register> &UpdRegs); bool isIntReg(RegisterRef RR, unsigned &BW); bool isIntraBlocks(LiveInterval &LI); bool coalesceRegisters(RegisterRef R1, RegisterRef R2); - bool coalesceSegments(const SmallVectorImpl<MachineInstr*> &Condsets, - std::set<unsigned> &UpdRegs); + bool coalesceSegments(const SmallVectorImpl<MachineInstr *> &Condsets, + std::set<Register> &UpdRegs); }; } // end anonymous namespace @@ -285,8 +286,8 @@ bool HexagonExpandCondsets::isCondset(const MachineInstr &MI) { return false; } -LaneBitmask HexagonExpandCondsets::getLaneMask(unsigned Reg, unsigned Sub) { - assert(Register::isVirtualRegister(Reg)); +LaneBitmask HexagonExpandCondsets::getLaneMask(Register Reg, unsigned Sub) { + assert(Reg.isVirtual()); return Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : MRI->getMaxLaneMaskForVReg(Reg); } @@ -312,7 +313,7 @@ bool HexagonExpandCondsets::isRefInMap(RegisterRef RR, ReferenceMap &Map, return false; } -void HexagonExpandCondsets::updateKillFlags(unsigned Reg) { +void HexagonExpandCondsets::updateKillFlags(Register Reg) { auto KillAt = [this,Reg] (SlotIndex K, LaneBitmask LM) -> void { // Set the <kill> flag on a use of Reg whose lane mask is contained in LM. MachineInstr *MI = LIS->getInstructionFromIndex(K); @@ -363,9 +364,9 @@ void HexagonExpandCondsets::updateKillFlags(unsigned Reg) { } } -void HexagonExpandCondsets::updateDeadsInRange(unsigned Reg, LaneBitmask LM, - LiveRange &Range) { - assert(Register::isVirtualRegister(Reg)); +void HexagonExpandCondsets::updateDeadsInRange(Register Reg, LaneBitmask LM, + LiveRange &Range) { + assert(Reg.isVirtual()); if (Range.empty()) return; @@ -374,7 +375,7 @@ void HexagonExpandCondsets::updateDeadsInRange(unsigned Reg, LaneBitmask LM, if (!Op.isReg() || !Op.isDef()) return { false, false }; Register DR = Op.getReg(), DSR = Op.getSubReg(); - if (!Register::isVirtualRegister(DR) || DR != Reg) + if (!DR.isVirtual() || DR != Reg) return { false, false }; LaneBitmask SLM = getLaneMask(DR, DSR); LaneBitmask A = SLM & LM; @@ -524,7 +525,7 @@ void HexagonExpandCondsets::updateDeadsInRange(unsigned Reg, LaneBitmask LM, } } -void HexagonExpandCondsets::updateDeadFlags(unsigned Reg) { +void HexagonExpandCondsets::updateDeadFlags(Register Reg) { LiveInterval &LI = LIS->getInterval(Reg); if (LI.hasSubRanges()) { for (LiveInterval::SubRange &S : LI.subranges()) { @@ -538,7 +539,7 @@ void HexagonExpandCondsets::updateDeadFlags(unsigned Reg) { } } -void HexagonExpandCondsets::recalculateLiveInterval(unsigned Reg) { +void HexagonExpandCondsets::recalculateLiveInterval(Register Reg) { LIS->removeInterval(Reg); LIS->createAndComputeVirtRegInterval(Reg); } @@ -548,12 +549,13 @@ void HexagonExpandCondsets::removeInstr(MachineInstr &MI) { MI.eraseFromParent(); } -void HexagonExpandCondsets::updateLiveness(std::set<unsigned> &RegSet, - bool Recalc, bool UpdateKills, bool UpdateDeads) { +void HexagonExpandCondsets::updateLiveness(std::set<Register> &RegSet, + bool Recalc, bool UpdateKills, + bool UpdateDeads) { UpdateKills |= UpdateDeads; - for (unsigned R : RegSet) { - if (!Register::isVirtualRegister(R)) { - assert(Register::isPhysicalRegister(R)); + for (Register R : RegSet) { + if (!R.isVirtual()) { + assert(R.isPhysical()); // There shouldn't be any physical registers as operands, except // possibly reserved registers. assert(MRI->isReserved(R)); @@ -580,17 +582,16 @@ unsigned HexagonExpandCondsets::getCondTfrOpcode(const MachineOperand &SO, using namespace Hexagon; if (SO.isReg()) { - Register PhysR; + MCRegister PhysR; RegisterRef RS = SO; - if (Register::isVirtualRegister(RS.Reg)) { + if (RS.Reg.isVirtual()) { const TargetRegisterClass *VC = MRI->getRegClass(RS.Reg); assert(VC->begin() != VC->end() && "Empty register class"); PhysR = *VC->begin(); } else { - assert(Register::isPhysicalRegister(RS.Reg)); PhysR = RS.Reg; } - Register PhysS = (RS.Sub == 0) ? PhysR : TRI->getSubReg(PhysR, RS.Sub); + MCRegister PhysS = (RS.Sub == 0) ? PhysR : TRI->getSubReg(PhysR, RS.Sub); const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysS); switch (TRI->getRegSizeInBits(*RC)) { case 32: @@ -661,7 +662,7 @@ MachineInstr *HexagonExpandCondsets::genCondTfrFor(MachineOperand &SrcOp, /// Replace a MUX instruction MI with a pair A2_tfrt/A2_tfrf. This function /// performs all necessary changes to complete the replacement. bool HexagonExpandCondsets::split(MachineInstr &MI, - std::set<unsigned> &UpdRegs) { + std::set<Register> &UpdRegs) { if (TfrLimitActive) { if (TfrCounter >= TfrLimit) return false; @@ -803,7 +804,7 @@ bool HexagonExpandCondsets::canMoveOver(MachineInstr &MI, ReferenceMap &Defs, // For physical register we would need to check register aliases, etc. // and we don't want to bother with that. It would be of little value // before the actual register rewriting (from virtual to physical). - if (!Register::isVirtualRegister(RR.Reg)) + if (!RR.Reg.isVirtual()) return false; // No redefs for any operand. if (isRefInMap(RR, Defs, Exec_Then)) @@ -855,7 +856,7 @@ void HexagonExpandCondsets::predicateAt(const MachineOperand &DefOp, MachineInstr &MI, MachineBasicBlock::iterator Where, const MachineOperand &PredOp, bool Cond, - std::set<unsigned> &UpdRegs) { + std::set<Register> &UpdRegs) { // The problem with updating live intervals is that we can move one def // past another def. In particular, this can happen when moving an A2_tfrt // over an A2_tfrf defining the same register. From the point of view of @@ -933,7 +934,7 @@ void HexagonExpandCondsets::renameInRange(RegisterRef RO, RegisterRef RN, /// the copy under the given condition (using the same predicate register as /// the copy). bool HexagonExpandCondsets::predicate(MachineInstr &TfrI, bool Cond, - std::set<unsigned> &UpdRegs) { + std::set<Register> &UpdRegs) { // TfrI - A2_tfr[tf] Instruction (not A2_tfrsi). unsigned Opc = TfrI.getOpcode(); (void)Opc; @@ -1000,7 +1001,7 @@ bool HexagonExpandCondsets::predicate(MachineInstr &TfrI, bool Cond, // subregisters are other physical registers, and we are not checking // that. RegisterRef RR = Op; - if (!Register::isVirtualRegister(RR.Reg)) + if (!RR.Reg.isVirtual()) return false; ReferenceMap &Map = Op.isDef() ? Defs : Uses; @@ -1067,7 +1068,7 @@ bool HexagonExpandCondsets::predicate(MachineInstr &TfrI, bool Cond, /// Predicate all cases of conditional copies in the specified block. bool HexagonExpandCondsets::predicateInBlock(MachineBasicBlock &B, - std::set<unsigned> &UpdRegs) { + std::set<Register> &UpdRegs) { bool Changed = false; MachineBasicBlock::iterator I, E, NextI; for (I = B.begin(), E = B.end(); I != E; I = NextI) { @@ -1092,7 +1093,7 @@ bool HexagonExpandCondsets::predicateInBlock(MachineBasicBlock &B, } bool HexagonExpandCondsets::isIntReg(RegisterRef RR, unsigned &BW) { - if (!Register::isVirtualRegister(RR.Reg)) + if (!RR.Reg.isVirtual()) return false; const TargetRegisterClass *RC = MRI->getRegClass(RR.Reg); if (RC == &Hexagon::IntRegsRegClass) { @@ -1172,7 +1173,7 @@ bool HexagonExpandCondsets::coalesceRegisters(RegisterRef R1, RegisterRef R2) { } L1.addSegment(LiveRange::Segment(I->start, I->end, NewVN)); } - while (L2.begin() != L2.end()) + while (!L2.empty()) L2.removeSegment(*L2.begin()); LIS->removeInterval(R2.Reg); @@ -1187,8 +1188,8 @@ bool HexagonExpandCondsets::coalesceRegisters(RegisterRef R1, RegisterRef R2) { /// the destination register. This could lead to having only one predicated /// instruction in the end instead of two. bool HexagonExpandCondsets::coalesceSegments( - const SmallVectorImpl<MachineInstr*> &Condsets, - std::set<unsigned> &UpdRegs) { + const SmallVectorImpl<MachineInstr *> &Condsets, + std::set<Register> &UpdRegs) { SmallVector<MachineInstr*,16> TwoRegs; for (MachineInstr *MI : Condsets) { MachineOperand &S1 = MI->getOperand(2), &S2 = MI->getOperand(3); @@ -1262,7 +1263,7 @@ bool HexagonExpandCondsets::runOnMachineFunction(MachineFunction &MF) { MF.getFunction().getParent())); bool Changed = false; - std::set<unsigned> CoalUpd, PredUpd; + std::set<Register> CoalUpd, PredUpd; SmallVector<MachineInstr*,16> Condsets; for (auto &B : MF) @@ -1279,7 +1280,7 @@ bool HexagonExpandCondsets::runOnMachineFunction(MachineFunction &MF) { // in the IR (they have been removed by live range analysis). // Updating them right before we split is the easiest, because splitting // adds definitions which would interfere with updating kills afterwards. - std::set<unsigned> KillUpd; + std::set<Register> KillUpd; for (MachineInstr *MI : Condsets) for (MachineOperand &Op : MI->operands()) if (Op.isReg() && Op.isUse()) diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp index 010b7171ce17..a62610ae2b7c 100644 --- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp @@ -306,7 +306,7 @@ static bool needsStackFrame(const MachineBasicBlock &MBB, const BitVector &CSR, Register R = MO.getReg(); // Virtual registers will need scavenging, which then may require // a stack slot. - if (Register::isVirtualRegister(R)) + if (R.isVirtual()) return true; for (MCSubRegIterator S(R, &HRI, true); S.isValid(); ++S) if (CSR[*S]) @@ -1104,7 +1104,8 @@ void HexagonFrameLowering::insertCFIInstructionsAt(MachineBasicBlock &MBB, Offset = MFI.getObjectOffset(F->getFrameIdx()); } else { Register FrameReg; - Offset = getFrameIndexReference(MF, F->getFrameIdx(), FrameReg); + Offset = + getFrameIndexReference(MF, F->getFrameIdx(), FrameReg).getFixed(); } // Subtract 8 to make room for R30 and R31, which are added above. Offset -= 8; @@ -1256,9 +1257,9 @@ static const char *getSpillFunctionFor(unsigned MaxReg, SpillKind SpillType, return nullptr; } -int HexagonFrameLowering::getFrameIndexReference(const MachineFunction &MF, - int FI, - Register &FrameReg) const { +StackOffset +HexagonFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, + Register &FrameReg) const { auto &MFI = MF.getFrameInfo(); auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); @@ -1354,7 +1355,7 @@ int HexagonFrameLowering::getFrameIndexReference(const MachineFunction &MF, int RealOffset = Offset; if (!UseFP && !UseAP) RealOffset = FrameSize+Offset; - return RealOffset; + return StackOffset::getFixed(RealOffset); } bool HexagonFrameLowering::insertCSRSpillsInBlock(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h index 87d385e1ce3c..4ffd31b670e4 100644 --- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h @@ -11,6 +11,7 @@ #include "Hexagon.h" #include "HexagonBlockRanges.h" +#include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -82,8 +83,8 @@ public: return true; } - int getFrameIndexReference(const MachineFunction &MF, int FI, - Register &FrameReg) const override; + StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, + Register &FrameReg) const override; bool hasFP(const MachineFunction &MF) const override; const SpillSlot *getCalleeSavedSpillSlots(unsigned &NumEntries) diff --git a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp index 2f29e88bc989..f2026877b22c 100644 --- a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp +++ b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp @@ -613,7 +613,7 @@ void HexagonGenInsert::buildOrderingMF(RegisterOrdering &RO) const { if (MO.isReg() && MO.isDef()) { Register R = MO.getReg(); assert(MO.getSubReg() == 0 && "Unexpected subregister in definition"); - if (Register::isVirtualRegister(R)) + if (R.isVirtual()) RO.insert(std::make_pair(R, Index++)); } } @@ -730,7 +730,7 @@ void HexagonGenInsert::getInstrDefs(const MachineInstr *MI, if (!MO.isReg() || !MO.isDef()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; Defs.insert(R); } @@ -743,7 +743,7 @@ void HexagonGenInsert::getInstrUses(const MachineInstr *MI, if (!MO.isReg() || !MO.isUse()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; Uses.insert(R); } @@ -1089,9 +1089,7 @@ void HexagonGenInsert::pruneCoveredSets(unsigned VR) { auto IsEmpty = [] (const IFRecordWithRegSet &IR) -> bool { return IR.second.empty(); }; - auto End = llvm::remove_if(LL, IsEmpty); - if (End != LL.end()) - LL.erase(End, LL.end()); + llvm::erase_if(LL, IsEmpty); } else { // The definition of VR is constant-extended, and all candidates have // empty removable-register sets. Pick the maximum candidate, and remove @@ -1179,9 +1177,7 @@ void HexagonGenInsert::pruneRegCopies(unsigned VR) { auto IsCopy = [] (const IFRecordWithRegSet &IR) -> bool { return IR.first.Wdh == 32 && (IR.first.Off == 0 || IR.first.Off == 32); }; - auto End = llvm::remove_if(LL, IsCopy); - if (End != LL.end()) - LL.erase(End, LL.end()); + llvm::erase_if(LL, IsCopy); } void HexagonGenInsert::pruneCandidates() { @@ -1483,7 +1479,7 @@ bool HexagonGenInsert::removeDeadCode(MachineDomTreeNode *N) { if (!MO.isReg() || !MO.isDef()) continue; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R) || !MRI->use_nodbg_empty(R)) { + if (!R.isVirtual() || !MRI->use_nodbg_empty(R)) { AllDead = false; break; } diff --git a/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp b/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp index 903287e68c99..d8d2025c5d27 100644 --- a/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp +++ b/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp @@ -48,7 +48,8 @@ namespace { // FIXME: Use TargetInstrInfo::RegSubRegPair struct RegisterSubReg { - unsigned R, S; + Register R; + unsigned S; RegisterSubReg(unsigned r = 0, unsigned s = 0) : R(r), S(s) {} RegisterSubReg(const MachineOperand &MO) : R(MO.getReg()), S(MO.getSubReg()) {} @@ -111,7 +112,7 @@ namespace { VectOfInst PUsers; RegToRegMap G2P; - bool isPredReg(unsigned R); + bool isPredReg(Register R); void collectPredicateGPR(MachineFunction &MF); void processPredicateGPR(const RegisterSubReg &Reg); unsigned getPredForm(unsigned Opc); @@ -133,8 +134,8 @@ INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_END(HexagonGenPredicate, "hexagon-gen-pred", "Hexagon generate predicate operations", false, false) -bool HexagonGenPredicate::isPredReg(unsigned R) { - if (!Register::isVirtualRegister(R)) +bool HexagonGenPredicate::isPredReg(Register R) { + if (!R.isVirtual()) return false; const TargetRegisterClass *RC = MRI->getRegClass(R); return RC == &Hexagon::PredRegsRegClass; @@ -214,7 +215,7 @@ void HexagonGenPredicate::collectPredicateGPR(MachineFunction &MF) { case TargetOpcode::COPY: if (isPredReg(MI->getOperand(1).getReg())) { RegisterSubReg RD = MI->getOperand(0); - if (Register::isVirtualRegister(RD.R)) + if (RD.R.isVirtual()) PredGPRs.insert(RD); } break; @@ -246,7 +247,7 @@ RegisterSubReg HexagonGenPredicate::getPredRegFor(const RegisterSubReg &Reg) { // Create a predicate register for a given Reg. The newly created register // will have its value copied from Reg, so that it can be later used as // an operand in other instructions. - assert(Register::isVirtualRegister(Reg.R)); + assert(Reg.R.isVirtual()); RegToRegMap::iterator F = G2P.find(Reg); if (F != G2P.end()) return F->second; @@ -472,9 +473,9 @@ bool HexagonGenPredicate::eliminatePredCopies(MachineFunction &MF) { continue; RegisterSubReg DR = MI.getOperand(0); RegisterSubReg SR = MI.getOperand(1); - if (!Register::isVirtualRegister(DR.R)) + if (!DR.R.isVirtual()) continue; - if (!Register::isVirtualRegister(SR.R)) + if (!SR.R.isVirtual()) continue; if (MRI->getRegClass(DR.R) != PredRC) continue; diff --git a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp index 4833935f8d24..2f23e8643720 100644 --- a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp +++ b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp @@ -390,7 +390,7 @@ bool HexagonHardwareLoops::runOnMachineFunction(MachineFunction &MF) { TRI = HST.getRegisterInfo(); for (auto &L : *MLI) - if (!L->getParentLoop()) { + if (L->isOutermost()) { bool L0Used = false; bool L1Used = false; Changed |= convertToHardwareLoop(L, L0Used, L1Used); @@ -1432,7 +1432,7 @@ bool HexagonHardwareLoops::loopCountMayWrapOrUnderFlow( Register Reg = InitVal->getReg(); // We don't know the value of a physical register. - if (!Register::isVirtualRegister(Reg)) + if (!Reg.isVirtual()) return true; MachineInstr *Def = MRI->getVRegDef(Reg); @@ -1510,7 +1510,7 @@ bool HexagonHardwareLoops::checkForImmediate(const MachineOperand &MO, int64_t TV; Register R = MO.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) return false; MachineInstr *DI = MRI->getVRegDef(R); unsigned DOpc = DI->getOpcode(); diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp index b4b389a7b956..bdd5c7dd151e 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp @@ -231,10 +231,10 @@ SDNode *HexagonDAGToDAGISel::StoreInstrForLoadIntrinsic(MachineSDNode *LoadN, if (Size >= 4) TS = CurDAG->getStore(SDValue(LoadN, 2), dl, SDValue(LoadN, 0), Loc, PI, - Size); + Align(Size)); else TS = CurDAG->getTruncStore(SDValue(LoadN, 2), dl, SDValue(LoadN, 0), Loc, - PI, MVT::getIntegerVT(Size * 8), Size); + PI, MVT::getIntegerVT(Size * 8), Align(Size)); SDNode *StoreN; { diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp index c0f92042e5da..29e76b53910e 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp @@ -789,6 +789,12 @@ struct ShuffleMask { OS << " }"; } }; + +LLVM_ATTRIBUTE_UNUSED +raw_ostream &operator<<(raw_ostream &OS, const ShuffleMask &SM) { + SM.print(OS); + return OS; +} } // namespace // -------------------------------------------------------------------- @@ -828,6 +834,7 @@ namespace llvm { void selectVAlign(SDNode *N); private: + void select(SDNode *ISelN); void materialize(const ResultStack &Results); SDValue getVectorConstant(ArrayRef<uint8_t> Data, const SDLoc &dl); @@ -931,46 +938,19 @@ bool HvxSelector::selectVectorConstants(SDNode *N) { SmallVector<SDNode*,4> Nodes; SetVector<SDNode*> WorkQ; - // The one-use test for VSPLATW's operand may fail due to dead nodes - // left over in the DAG. - DAG.RemoveDeadNodes(); - // The DAG can change (due to CSE) during selection, so cache all the // unselected nodes first to avoid traversing a mutating DAG. - - auto IsNodeToSelect = [] (SDNode *N) { - if (N->isMachineOpcode()) - return false; - switch (N->getOpcode()) { - case HexagonISD::VZERO: - case HexagonISD::VSPLATW: - return true; - case ISD::LOAD: { - SDValue Addr = cast<LoadSDNode>(N)->getBasePtr(); - unsigned AddrOpc = Addr.getOpcode(); - if (AddrOpc == HexagonISD::AT_PCREL || AddrOpc == HexagonISD::CP) - if (Addr.getOperand(0).getOpcode() == ISD::TargetConstantPool) - return true; - } - break; - } - // Make sure to select the operand of VSPLATW. - bool IsSplatOp = N->hasOneUse() && - N->use_begin()->getOpcode() == HexagonISD::VSPLATW; - return IsSplatOp; - }; - WorkQ.insert(N); for (unsigned i = 0; i != WorkQ.size(); ++i) { SDNode *W = WorkQ[i]; - if (IsNodeToSelect(W)) + if (!W->isMachineOpcode() && W->getOpcode() == HexagonISD::ISEL) Nodes.push_back(W); for (unsigned j = 0, f = W->getNumOperands(); j != f; ++j) WorkQ.insert(W->getOperand(j).getNode()); } for (SDNode *L : Nodes) - ISel.Select(L); + select(L); return !Nodes.empty(); } @@ -1358,6 +1338,82 @@ namespace { }; } +void HvxSelector::select(SDNode *ISelN) { + // What's important here is to select the right set of nodes. The main + // selection algorithm loops over nodes in a topological order, i.e. users + // are visited before their operands. + // + // It is an error to have an unselected node with a selected operand, and + // there is an assertion in the main selector code to enforce that. + // + // Such a situation could occur if we selected a node, which is both a + // subnode of ISelN, and a subnode of an unrelated (and yet unselected) + // node in the DAG. + assert(ISelN->getOpcode() == HexagonISD::ISEL); + SDNode *N0 = ISelN->getOperand(0).getNode(); + if (N0->isMachineOpcode()) { + ISel.ReplaceNode(ISelN, N0); + return; + } + + // There could have been nodes created (i.e. inserted into the DAG) + // that are now dead. Remove them, in case they use any of the nodes + // to select (and make them look shared). + DAG.RemoveDeadNodes(); + + SetVector<SDNode*> SubNodes, TmpQ; + std::map<SDNode*,unsigned> NumOps; + + // Don't want to select N0 if it's shared with another node, except if + // it's shared with other ISELs. + auto IsISelN = [](SDNode *T) { return T->getOpcode() == HexagonISD::ISEL; }; + if (llvm::all_of(N0->uses(), IsISelN)) + SubNodes.insert(N0); + + auto InSubNodes = [&SubNodes](SDNode *T) { return SubNodes.count(T); }; + for (unsigned I = 0; I != SubNodes.size(); ++I) { + SDNode *S = SubNodes[I]; + unsigned OpN = 0; + // Only add subnodes that are only reachable from N0. + for (SDValue Op : S->ops()) { + SDNode *O = Op.getNode(); + if (llvm::all_of(O->uses(), InSubNodes)) { + SubNodes.insert(O); + ++OpN; + } + } + NumOps.insert({S, OpN}); + if (OpN == 0) + TmpQ.insert(S); + } + + for (unsigned I = 0; I != TmpQ.size(); ++I) { + SDNode *S = TmpQ[I]; + for (SDNode *U : S->uses()) { + if (U == ISelN) + continue; + auto F = NumOps.find(U); + assert(F != NumOps.end()); + if (F->second > 0 && !--F->second) + TmpQ.insert(F->first); + } + } + + // Remove the marker. + ISel.ReplaceNode(ISelN, N0); + + assert(SubNodes.size() == TmpQ.size()); + NullifyingVector<decltype(TmpQ)::vector_type> Queue(TmpQ.takeVector()); + + Deleter DUQ(DAG, Queue); + for (SDNode *S : reverse(Queue)) { + if (S == nullptr) + continue; + DEBUG_WITH_TYPE("isel", {dbgs() << "HVX selecting: "; S->dump(&DAG);}); + ISel.Select(S); + } +} + bool HvxSelector::scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl, MVT ResTy, SDValue Va, SDValue Vb, SDNode *N) { @@ -1379,12 +1435,7 @@ bool HvxSelector::scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl, // nodes, these nodes would not be selected (since the "local" selection // only visits nodes that are not in AllNodes). // To avoid this issue, remove all dead nodes from the DAG now. - DAG.RemoveDeadNodes(); - DenseSet<SDNode*> AllNodes; - for (SDNode &S : DAG.allnodes()) - AllNodes.insert(&S); - - Deleter DUA(DAG, AllNodes); +// DAG.RemoveDeadNodes(); SmallVector<SDValue,128> Ops; LLVMContext &Ctx = *DAG.getContext(); @@ -1434,57 +1485,9 @@ bool HvxSelector::scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl, } assert(!N->use_empty()); - ISel.ReplaceNode(N, LV.getNode()); - - if (AllNodes.count(LV.getNode())) { - DAG.RemoveDeadNodes(); - return true; - } - - // The lowered build-vector node will now need to be selected. It needs - // to be done here because this node and its submodes are not included - // in the main selection loop. - // Implement essentially the same topological ordering algorithm as is - // used in SelectionDAGISel. - - SetVector<SDNode*> SubNodes, TmpQ; - std::map<SDNode*,unsigned> NumOps; - - SubNodes.insert(LV.getNode()); - for (unsigned I = 0; I != SubNodes.size(); ++I) { - unsigned OpN = 0; - SDNode *S = SubNodes[I]; - for (SDValue Op : S->ops()) { - if (AllNodes.count(Op.getNode())) - continue; - SubNodes.insert(Op.getNode()); - ++OpN; - } - NumOps.insert({S, OpN}); - if (OpN == 0) - TmpQ.insert(S); - } - - for (unsigned I = 0; I != TmpQ.size(); ++I) { - SDNode *S = TmpQ[I]; - for (SDNode *U : S->uses()) { - if (!SubNodes.count(U)) - continue; - auto F = NumOps.find(U); - assert(F != NumOps.end()); - assert(F->second > 0); - if (!--F->second) - TmpQ.insert(F->first); - } - } - assert(SubNodes.size() == TmpQ.size()); - NullifyingVector<decltype(TmpQ)::vector_type> Queue(TmpQ.takeVector()); - - Deleter DUQ(DAG, Queue); - for (SDNode *S : reverse(Queue)) - if (S != nullptr) - ISel.Select(S); - + SDValue IS = DAG.getNode(HexagonISD::ISEL, dl, ResTy, LV); + ISel.ReplaceNode(N, IS.getNode()); + select(IS.getNode()); DAG.RemoveDeadNodes(); return true; } @@ -1683,7 +1686,7 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) { // The result length must be the same as the length of a single vector, // or a vector pair. assert(LogLen == HwLog || LogLen == HwLog+1); - bool Extend = (LogLen == HwLog); + bool HavePairs = LogLen == HwLog+1; if (!isPermutation(SM.Mask)) return OpRef::fail(); @@ -1767,6 +1770,22 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) { // E 1 1 1 0 7 0 1 1 1 7 0 1 1 1 7 0 1 1 1 // F 1 1 1 1 F 1 1 1 1 F 1 1 1 1 F 1 1 1 1 + // There is one special case that is not a perfect shuffle, but + // can be turned into one easily: when the shuffle operates on + // a vector pair, but the two vectors in the pair are swapped. + // The code below that identifies perfect shuffles will reject + // it, unless the order is reversed. + SmallVector<int,128> MaskStorage(SM.Mask.begin(), SM.Mask.end()); + bool InvertedPair = false; + if (HavePairs && SM.Mask[0] >= int(HwLen)) { + for (int i = 0, e = SM.Mask.size(); i != e; ++i) { + int M = SM.Mask[i]; + MaskStorage[i] = M >= int(HwLen) ? M-HwLen : M+HwLen; + } + InvertedPair = true; + } + ArrayRef<int> LocalMask(MaskStorage); + auto XorPow2 = [] (ArrayRef<int> Mask, unsigned Num) { unsigned X = Mask[0] ^ Mask[Num/2]; // Check that the first half has the X's bits clear. @@ -1786,12 +1805,12 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) { assert(VecLen > 2); for (unsigned I = VecLen; I >= 2; I >>= 1) { // Examine the initial segment of Mask of size I. - unsigned X = XorPow2(SM.Mask, I); + unsigned X = XorPow2(LocalMask, I); if (!isPowerOf2_32(X)) return OpRef::fail(); // Check the other segments of Mask. for (int J = I; J < VecLen; J += I) { - if (XorPow2(SM.Mask.slice(J, I), I) != X) + if (XorPow2(LocalMask.slice(J, I), I) != X) return OpRef::fail(); } Perm[Log2_32(X)] = Log2_32(I)-1; @@ -1895,20 +1914,40 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) { } } + // From the cycles, construct the sequence of values that will + // then form the control values for vdealvdd/vshuffvdd, i.e. + // (M a1 a2)(M a3 a4 a5)... -> a1 a2 a3 a4 a5 + // This essentially strips the M value from the cycles where + // it's present, and performs the insertion of M (then stripping) + // for cycles without M (as described in an earlier comment). SmallVector<unsigned,8> SwapElems; - if (HwLen == unsigned(VecLen)) + // When the input is extended (i.e. single vector becomes a pair), + // this is done by using an "undef" vector as the second input. + // However, then we get + // input 1: GOODBITS + // input 2: ........ + // but we need + // input 1: ....BITS + // input 2: ....GOOD + // Then at the end, this needs to be undone. To accomplish this, + // artificially add "LogLen-1" at both ends of the sequence. + if (!HavePairs) SwapElems.push_back(LogLen-1); - for (const CycleType &C : Cycles) { + // Do the transformation: (a1..an) -> (M a1..an)(M a1). unsigned First = (C[0] == LogLen-1) ? 1 : 0; SwapElems.append(C.begin()+First, C.end()); if (First == 0) SwapElems.push_back(C[0]); } + if (!HavePairs) + SwapElems.push_back(LogLen-1); const SDLoc &dl(Results.InpNode); - OpRef Arg = !Extend ? Va - : concat(Va, OpRef::undef(SingleTy), Results); + OpRef Arg = HavePairs ? Va + : concat(Va, OpRef::undef(SingleTy), Results); + if (InvertedPair) + Arg = concat(OpRef::hi(Arg), OpRef::lo(Arg), Results); for (unsigned I = 0, E = SwapElems.size(); I != E; ) { bool IsInc = I == E-1 || SwapElems[I] < SwapElems[I+1]; @@ -1932,7 +1971,7 @@ OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) { Arg = OpRef::res(Results.top()); } - return !Extend ? Arg : OpRef::lo(Arg); + return HavePairs ? Arg : OpRef::lo(Arg); } OpRef HvxSelector::butterfly(ShuffleMask SM, OpRef Va, ResultStack &Results) { @@ -1996,7 +2035,7 @@ SDValue HvxSelector::getVectorConstant(ArrayRef<uint8_t> Data, SDValue BV = DAG.getBuildVector(VecTy, dl, Elems); SDValue LV = Lower.LowerOperation(BV, DAG); DAG.RemoveDeadNode(BV.getNode()); - return LV; + return DAG.getNode(HexagonISD::ISEL, dl, VecTy, LV); } void HvxSelector::selectShuffle(SDNode *N) { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 768fea639cf9..c8994a3a28a3 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -1517,8 +1517,11 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max()); setOperationAction(ISD::BR_JT, MVT::Other, Expand); - setOperationAction(ISD::ABS, MVT::i32, Legal); - setOperationAction(ISD::ABS, MVT::i64, Legal); + for (unsigned LegalIntOp : + {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) { + setOperationAction(LegalIntOp, MVT::i32, Legal); + setOperationAction(LegalIntOp, MVT::i64, Legal); + } // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit, // but they only operate on i64. @@ -1620,7 +1623,8 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR, - ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE + ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE, + ISD::SPLAT_VECTOR, }; for (MVT VT : MVT::fixedlen_vector_valuetypes()) { @@ -1677,6 +1681,16 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, setOperationAction(ISD::AND, NativeVT, Legal); setOperationAction(ISD::OR, NativeVT, Legal); setOperationAction(ISD::XOR, NativeVT, Legal); + + if (NativeVT.getVectorElementType() != MVT::i1) + setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal); + } + + for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) { + setOperationAction(ISD::SMIN, VT, Legal); + setOperationAction(ISD::SMAX, VT, Legal); + setOperationAction(ISD::UMIN, VT, Legal); + setOperationAction(ISD::UMAX, VT, Legal); } // Custom lower unaligned loads. @@ -1843,15 +1857,12 @@ const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { case HexagonISD::VASL: return "HexagonISD::VASL"; case HexagonISD::VASR: return "HexagonISD::VASR"; case HexagonISD::VLSR: return "HexagonISD::VLSR"; - case HexagonISD::VSPLAT: return "HexagonISD::VSPLAT"; case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW"; case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0"; case HexagonISD::VROR: return "HexagonISD::VROR"; case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE"; case HexagonISD::PTRUE: return "HexagonISD::PTRUE"; case HexagonISD::PFALSE: return "HexagonISD::PFALSE"; - case HexagonISD::VZERO: return "HexagonISD::VZERO"; - case HexagonISD::VSPLATW: return "HexagonISD::VSPLATW"; case HexagonISD::D2P: return "HexagonISD::D2P"; case HexagonISD::P2D: return "HexagonISD::P2D"; case HexagonISD::V2Q: return "HexagonISD::V2Q"; @@ -1862,6 +1873,10 @@ const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST"; case HexagonISD::VALIGN: return "HexagonISD::VALIGN"; case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR"; + case HexagonISD::VPACKL: return "HexagonISD::VPACKL"; + case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK"; + case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU"; + case HexagonISD::ISEL: return "HexagonISD::ISEL"; case HexagonISD::OP_END: break; } return nullptr; @@ -2064,20 +2079,9 @@ HexagonTargetLowering::getPreferredVectorAction(MVT VT) const { return TargetLoweringBase::TypeScalarizeVector; if (Subtarget.useHVXOps()) { - unsigned HwLen = Subtarget.getVectorLength(); - // If the size of VT is at least half of the vector length, - // widen the vector. Note: the threshold was not selected in - // any scientific way. - ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes(); - if (llvm::find(Tys, ElemTy) != Tys.end()) { - unsigned HwWidth = 8*HwLen; - unsigned VecWidth = VT.getSizeInBits(); - if (VecWidth >= HwWidth/2 && VecWidth < HwWidth) - return TargetLoweringBase::TypeWidenVector; - } - // Split vectors of i1 that correspond to (byte) vector pairs. - if (ElemTy == MVT::i1 && VecLen == 2*HwLen) - return TargetLoweringBase::TypeSplitVector; + unsigned Action = getPreferredHvxVectorAction(VT); + if (Action != ~0u) + return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action); } // Always widen (remaining) vectors of i1. @@ -2229,26 +2233,33 @@ HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) SDValue HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const { - if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) { - if (SDValue S = BVN->getSplatValue()) { - unsigned NewOpc; - switch (Op.getOpcode()) { - case ISD::SHL: - NewOpc = HexagonISD::VASL; - break; - case ISD::SRA: - NewOpc = HexagonISD::VASR; - break; - case ISD::SRL: - NewOpc = HexagonISD::VLSR; - break; - default: - llvm_unreachable("Unexpected shift opcode"); - } - return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), S); - } + unsigned NewOpc; + switch (Op.getOpcode()) { + case ISD::SHL: + NewOpc = HexagonISD::VASL; + break; + case ISD::SRA: + NewOpc = HexagonISD::VASR; + break; + case ISD::SRL: + NewOpc = HexagonISD::VLSR; + break; + default: + llvm_unreachable("Unexpected shift opcode"); } + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); + const SDLoc &dl(Op); + + switch (Op1.getOpcode()) { + case ISD::BUILD_VECTOR: + if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue()) + return DAG.getNode(NewOpc, dl, ty(Op), Op0, S); + break; + case ISD::SPLAT_VECTOR: + return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0)); + } return SDValue(); } @@ -2325,9 +2336,10 @@ HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); unsigned First, Num = Elem.size(); - for (First = 0; First != Num; ++First) + for (First = 0; First != Num; ++First) { if (!isUndef(Elem[First])) break; + } if (First == Num) return DAG.getUNDEF(VecTy); @@ -2359,18 +2371,16 @@ HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, // Then try splat. bool IsSplat = true; - for (unsigned i = 0; i != Num; ++i) { - if (i == First) - continue; + for (unsigned i = First+1; i != Num; ++i) { if (Elem[i] == Elem[First] || isUndef(Elem[i])) continue; IsSplat = false; break; } if (IsSplat) { - // Legalize the operand to VSPLAT. + // Legalize the operand of SPLAT_VECTOR. SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); - return DAG.getNode(HexagonISD::VSPLAT, dl, VecTy, Ext); + return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); } // Generate @@ -2408,9 +2418,10 @@ HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); unsigned First, Num = Elem.size(); - for (First = 0; First != Num; ++First) + for (First = 0; First != Num; ++First) { if (!isUndef(Elem[First])) break; + } if (First == Num) return DAG.getUNDEF(VecTy); @@ -2421,18 +2432,16 @@ HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, // First try splat if possible. if (ElemTy == MVT::i16) { bool IsSplat = true; - for (unsigned i = 0; i != Num; ++i) { - if (i == First) - continue; + for (unsigned i = First+1; i != Num; ++i) { if (Elem[i] == Elem[First] || isUndef(Elem[i])) continue; IsSplat = false; break; } if (IsSplat) { - // Legalize the operand to VSPLAT. + // Legalize the operand of SPLAT_VECTOR SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); - return DAG.getNode(HexagonISD::VSPLAT, dl, VecTy, Ext); + return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); } } @@ -2650,7 +2659,7 @@ HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) unsigned W = Ty.getSizeInBits(); if (W <= 64) return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W))); - return DAG.getNode(HexagonISD::VZERO, dl, Ty); + return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG)); } if (Ty.isInteger()) @@ -2661,6 +2670,28 @@ HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) } SDValue +HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG) + const { + MVT ValTy = ty(Val); + assert(ValTy.getVectorElementType() == ResTy.getVectorElementType()); + + unsigned ValLen = ValTy.getVectorNumElements(); + unsigned ResLen = ResTy.getVectorNumElements(); + if (ValLen == ResLen) + return Val; + + const SDLoc &dl(Val); + assert(ValLen < ResLen); + assert(ResLen % ValLen == 0); + + SmallVector<SDValue, 4> Concats = {Val}; + for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i) + Concats.push_back(DAG.getUNDEF(ValTy)); + + return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats); +} + +SDValue HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { MVT VecTy = ty(Op); unsigned BW = VecTy.getSizeInBits(); @@ -2910,8 +2941,10 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first, DAG.getConstant(NeedAlign, dl, MVT::i32)) : BO.first; - SDValue Base0 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second, dl); - SDValue Base1 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second+LoadLen, dl); + SDValue Base0 = + DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl); + SDValue Base1 = DAG.getMemBasePlusOffset( + BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl); MachineMemOperand *WideMMO = nullptr; if (MachineMemOperand *MMO = LN->getMemOperand()) { @@ -3023,7 +3056,7 @@ HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR) return LowerINLINEASM(Op, DAG); - if (isHvxOperation(Op)) { + if (isHvxOperation(Op.getNode(), DAG)) { // If HVX lowering returns nothing, try the default lowering. if (SDValue V = LowerHvxOperation(Op, DAG)) return V; @@ -3084,7 +3117,7 @@ void HexagonTargetLowering::LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { - if (isHvxOperation(N)) { + if (isHvxOperation(N, DAG)) { LowerHvxOperationWrapper(N, Results, DAG); if (!Results.empty()) return; @@ -3103,7 +3136,7 @@ void HexagonTargetLowering::ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { - if (isHvxOperation(N)) { + if (isHvxOperation(N, DAG)) { ReplaceHvxNodeResults(N, Results, DAG); if (!Results.empty()) return; @@ -3118,10 +3151,12 @@ HexagonTargetLowering::ReplaceNodeResults(SDNode *N, case ISD::BITCAST: // Handle a bitcast from v8i1 to i8. if (N->getValueType(0) == MVT::i8) { - SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, - N->getOperand(0), DAG); - SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8); - Results.push_back(T); + if (N->getOperand(0).getValueType() == MVT::v8i1) { + SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, + N->getOperand(0), DAG); + SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8); + Results.push_back(T); + } } break; } @@ -3130,13 +3165,16 @@ HexagonTargetLowering::ReplaceNodeResults(SDNode *N, SDValue HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { - SDValue Op(N, 0); - if (isHvxOperation(Op)) { + if (isHvxOperation(N, DCI.DAG)) { if (SDValue V = PerformHvxDAGCombine(N, DCI)) return V; return SDValue(); } + if (DCI.isBeforeLegalizeOps()) + return SDValue(); + + SDValue Op(N, 0); const SDLoc &dl(Op); unsigned Opc = Op.getOpcode(); diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index 7d6e6b6185c8..cfccb14a09c9 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -15,6 +15,7 @@ #define LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H #include "Hexagon.h" +#include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -30,464 +31,478 @@ namespace llvm { namespace HexagonISD { - enum NodeType : unsigned { - OP_BEGIN = ISD::BUILTIN_OP_END, +enum NodeType : unsigned { + OP_BEGIN = ISD::BUILTIN_OP_END, - CONST32 = OP_BEGIN, - CONST32_GP, // For marking data present in GP. - ADDC, // Add with carry: (X, Y, Cin) -> (X+Y, Cout). - SUBC, // Sub with carry: (X, Y, Cin) -> (X+~Y+Cin, Cout). - ALLOCA, + CONST32 = OP_BEGIN, + CONST32_GP, // For marking data present in GP. + ADDC, // Add with carry: (X, Y, Cin) -> (X+Y, Cout). + SUBC, // Sub with carry: (X, Y, Cin) -> (X+~Y+Cin, Cout). + ALLOCA, - AT_GOT, // Index in GOT. - AT_PCREL, // Offset relative to PC. + AT_GOT, // Index in GOT. + AT_PCREL, // Offset relative to PC. - CALL, // Function call. - CALLnr, // Function call that does not return. - CALLR, + CALL, // Function call. + CALLnr, // Function call that does not return. + CALLR, - RET_FLAG, // Return with a flag operand. - BARRIER, // Memory barrier. - JT, // Jump table. - CP, // Constant pool. + RET_FLAG, // Return with a flag operand. + BARRIER, // Memory barrier. + JT, // Jump table. + CP, // Constant pool. - COMBINE, - VSPLAT, // Generic splat, selection depends on argument/return - // types. - VASL, - VASR, - VLSR, + COMBINE, + VASL, + VASR, + VLSR, - TSTBIT, - INSERT, - EXTRACTU, - VEXTRACTW, - VINSERTW0, - VROR, - TC_RETURN, - EH_RETURN, - DCFETCH, - READCYCLE, - PTRUE, - PFALSE, - D2P, // Convert 8-byte value to 8-bit predicate register. [*] - P2D, // Convert 8-bit predicate register to 8-byte value. [*] - V2Q, // Convert HVX vector to a vector predicate reg. [*] - Q2V, // Convert vector predicate to an HVX vector. [*] - // [*] The equivalence is defined as "Q <=> (V != 0)", - // where the != operation compares bytes. - // Note: V != 0 is implemented as V >u 0. - QCAT, - QTRUE, - QFALSE, - VZERO, - VSPLATW, // HVX splat of a 32-bit word with an arbitrary result type. - TYPECAST, // No-op that's used to convert between different legal - // types in a register. - VALIGN, // Align two vectors (in Op0, Op1) to one that would have - // been loaded from address in Op2. - VALIGNADDR, // Align vector address: Op0 & -Op1, except when it is - // an address in a vector load, then it's a no-op. - OP_END - }; + TSTBIT, + INSERT, + EXTRACTU, + VEXTRACTW, + VINSERTW0, + VROR, + TC_RETURN, + EH_RETURN, + DCFETCH, + READCYCLE, + PTRUE, + PFALSE, + D2P, // Convert 8-byte value to 8-bit predicate register. [*] + P2D, // Convert 8-bit predicate register to 8-byte value. [*] + V2Q, // Convert HVX vector to a vector predicate reg. [*] + Q2V, // Convert vector predicate to an HVX vector. [*] + // [*] The equivalence is defined as "Q <=> (V != 0)", + // where the != operation compares bytes. + // Note: V != 0 is implemented as V >u 0. + QCAT, + QTRUE, + QFALSE, + TYPECAST, // No-op that's used to convert between different legal + // types in a register. + VALIGN, // Align two vectors (in Op0, Op1) to one that would have + // been loaded from address in Op2. + VALIGNADDR, // Align vector address: Op0 & -Op1, except when it is + // an address in a vector load, then it's a no-op. + VPACKL, // Pack low parts of the input vector to the front of the + // output. For example v64i16 VPACKL(v32i32) will pick + // the low halfwords and pack them into the first 32 + // halfwords of the output. The rest of the output is + // unspecified. + VUNPACK, // Unpacking into low elements with sign extension. + VUNPACKU, // Unpacking into low elements with zero extension. + ISEL, // Marker for nodes that were created during ISel, and + // which need explicit selection (would have been left + // unselected otherwise). + OP_END +}; } // end namespace HexagonISD - class HexagonSubtarget; +class HexagonSubtarget; - class HexagonTargetLowering : public TargetLowering { - int VarArgsFrameOffset; // Frame offset to start of varargs area. - const HexagonTargetMachine &HTM; - const HexagonSubtarget &Subtarget; +class HexagonTargetLowering : public TargetLowering { + int VarArgsFrameOffset; // Frame offset to start of varargs area. + const HexagonTargetMachine &HTM; + const HexagonSubtarget &Subtarget; - bool CanReturnSmallStruct(const Function* CalleeFn, unsigned& RetSize) - const; + bool CanReturnSmallStruct(const Function* CalleeFn, unsigned& RetSize) + const; - public: - explicit HexagonTargetLowering(const TargetMachine &TM, - const HexagonSubtarget &ST); +public: + explicit HexagonTargetLowering(const TargetMachine &TM, + const HexagonSubtarget &ST); - bool isHVXVectorType(MVT Ty) const; + bool isHVXVectorType(MVT Ty) const; - /// IsEligibleForTailCallOptimization - Check whether the call is eligible - /// for tail call optimization. Targets which want to do tail call - /// optimization should implement this function. - bool IsEligibleForTailCallOptimization(SDValue Callee, - CallingConv::ID CalleeCC, bool isVarArg, bool isCalleeStructRet, - bool isCallerStructRet, const SmallVectorImpl<ISD::OutputArg> &Outs, - const SmallVectorImpl<SDValue> &OutVals, - const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const; + /// IsEligibleForTailCallOptimization - Check whether the call is eligible + /// for tail call optimization. Targets which want to do tail call + /// optimization should implement this function. + bool IsEligibleForTailCallOptimization(SDValue Callee, + CallingConv::ID CalleeCC, bool isVarArg, bool isCalleeStructRet, + bool isCallerStructRet, const SmallVectorImpl<ISD::OutputArg> &Outs, + const SmallVectorImpl<SDValue> &OutVals, + const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, - MachineFunction &MF, - unsigned Intrinsic) const override; + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + MachineFunction &MF, + unsigned Intrinsic) const override; - bool isTruncateFree(Type *Ty1, Type *Ty2) const override; - bool isTruncateFree(EVT VT1, EVT VT2) const override; + bool isTruncateFree(Type *Ty1, Type *Ty2) const override; + bool isTruncateFree(EVT VT1, EVT VT2) const override; - bool isCheapToSpeculateCttz() const override { return true; } - bool isCheapToSpeculateCtlz() const override { return true; } - bool isCtlzFast() const override { return true; } + bool isCheapToSpeculateCttz() const override { return true; } + bool isCheapToSpeculateCtlz() const override { return true; } + bool isCtlzFast() const override { return true; } - bool hasBitTest(SDValue X, SDValue Y) const override; + bool hasBitTest(SDValue X, SDValue Y) const override; - bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override; + bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override; - /// Return true if an FMA operation is faster than a pair of mul and add - /// instructions. fmuladd intrinsics will be expanded to FMAs when this - /// method returns true (and FMAs are legal), otherwise fmuladd is - /// expanded to mul + add. - bool isFMAFasterThanFMulAndFAdd(const MachineFunction &, - EVT) const override; + /// Return true if an FMA operation is faster than a pair of mul and add + /// instructions. fmuladd intrinsics will be expanded to FMAs when this + /// method returns true (and FMAs are legal), otherwise fmuladd is + /// expanded to mul + add. + bool isFMAFasterThanFMulAndFAdd(const MachineFunction &, + EVT) const override; - // Should we expand the build vector with shuffles? - bool shouldExpandBuildVectorWithShuffles(EVT VT, - unsigned DefinedValues) const override; + // Should we expand the build vector with shuffles? + bool shouldExpandBuildVectorWithShuffles(EVT VT, + unsigned DefinedValues) const override; - bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override; - TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) - const override; + bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override; + TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) + const override; - SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; - void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) const override; - void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) const override; + SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; + void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, + SelectionDAG &DAG) const override; + void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, + SelectionDAG &DAG) const override; - const char *getTargetNodeName(unsigned Opcode) const override; + const char *getTargetNodeName(unsigned Opcode) const override; - SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const; - SDValue - LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, - const SmallVectorImpl<ISD::InputArg> &Ins, - const SDLoc &dl, SelectionDAG &DAG, - SmallVectorImpl<SDValue> &InVals) const override; - SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, - SelectionDAG &DAG) const; - SDValue LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, - SelectionDAG &DAG) const; - SDValue LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, - SelectionDAG &DAG) const; - SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, - GlobalAddressSDNode *GA, SDValue InFlag, EVT PtrVT, - unsigned ReturnReg, unsigned char OperandFlags) const; - SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const; + SDValue + LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, + const SmallVectorImpl<ISD::InputArg> &Ins, + const SDLoc &dl, SelectionDAG &DAG, + SmallVectorImpl<SDValue> &InVals) const override; + SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, + SelectionDAG &DAG) const; + SDValue LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, + SelectionDAG &DAG) const; + SDValue LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, + SelectionDAG &DAG) const; + SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, + GlobalAddressSDNode *GA, SDValue InFlag, EVT PtrVT, + unsigned ReturnReg, unsigned char OperandFlags) const; + SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl<SDValue> &InVals) const override; - SDValue LowerCallResult(SDValue Chain, SDValue InFlag, - CallingConv::ID CallConv, bool isVarArg, - const SmallVectorImpl<ISD::InputArg> &Ins, - const SDLoc &dl, SelectionDAG &DAG, - SmallVectorImpl<SDValue> &InVals, - const SmallVectorImpl<SDValue> &OutVals, - SDValue Callee) const; + SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, + SmallVectorImpl<SDValue> &InVals) const override; + SDValue LowerCallResult(SDValue Chain, SDValue InFlag, + CallingConv::ID CallConv, bool isVarArg, + const SmallVectorImpl<ISD::InputArg> &Ins, + const SDLoc &dl, SelectionDAG &DAG, + SmallVectorImpl<SDValue> &InVals, + const SmallVectorImpl<SDValue> &OutVals, + SDValue Callee) const; - SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const; - SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const; + SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; - bool CanLowerReturn(CallingConv::ID CallConv, - MachineFunction &MF, bool isVarArg, - const SmallVectorImpl<ISD::OutputArg> &Outs, - LLVMContext &Context) const override; + bool CanLowerReturn(CallingConv::ID CallConv, + MachineFunction &MF, bool isVarArg, + const SmallVectorImpl<ISD::OutputArg> &Outs, + LLVMContext &Context) const override; - SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, - const SmallVectorImpl<ISD::OutputArg> &Outs, - const SmallVectorImpl<SDValue> &OutVals, - const SDLoc &dl, SelectionDAG &DAG) const override; + SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, + const SmallVectorImpl<ISD::OutputArg> &Outs, + const SmallVectorImpl<SDValue> &OutVals, + const SDLoc &dl, SelectionDAG &DAG) const override; - SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; + SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; - bool mayBeEmittedAsTailCall(const CallInst *CI) const override; + bool mayBeEmittedAsTailCall(const CallInst *CI) const override; - Register getRegisterByName(const char* RegName, LLT VT, - const MachineFunction &MF) const override; + Register getRegisterByName(const char* RegName, LLT VT, + const MachineFunction &MF) const override; - /// If a physical register, this returns the register that receives the - /// exception address on entry to an EH pad. - Register - getExceptionPointerRegister(const Constant *PersonalityFn) const override { - return Hexagon::R0; - } + /// If a physical register, this returns the register that receives the + /// exception address on entry to an EH pad. + Register + getExceptionPointerRegister(const Constant *PersonalityFn) const override { + return Hexagon::R0; + } - /// If a physical register, this returns the register that receives the - /// exception typeid on entry to a landing pad. - Register - getExceptionSelectorRegister(const Constant *PersonalityFn) const override { - return Hexagon::R1; - } + /// If a physical register, this returns the register that receives the + /// exception typeid on entry to a landing pad. + Register + getExceptionSelectorRegister(const Constant *PersonalityFn) const override { + return Hexagon::R1; + } - SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; - EVT getSetCCResultType(const DataLayout &, LLVMContext &C, - EVT VT) const override { - if (!VT.isVector()) - return MVT::i1; - else - return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); - } + EVT getSetCCResultType(const DataLayout &, LLVMContext &C, + EVT VT) const override { + if (!VT.isVector()) + return MVT::i1; + else + return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); + } - bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, - SDValue &Base, SDValue &Offset, - ISD::MemIndexedMode &AM, - SelectionDAG &DAG) const override; + bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, + SDValue &Base, SDValue &Offset, + ISD::MemIndexedMode &AM, + SelectionDAG &DAG) const override; - ConstraintType getConstraintType(StringRef Constraint) const override; + ConstraintType getConstraintType(StringRef Constraint) const override; - std::pair<unsigned, const TargetRegisterClass *> - getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, - StringRef Constraint, MVT VT) const override; + std::pair<unsigned, const TargetRegisterClass *> + getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, + StringRef Constraint, MVT VT) const override; - unsigned - getInlineAsmMemConstraint(StringRef ConstraintCode) const override { - if (ConstraintCode == "o") - return InlineAsm::Constraint_o; - return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); - } + unsigned + getInlineAsmMemConstraint(StringRef ConstraintCode) const override { + if (ConstraintCode == "o") + return InlineAsm::Constraint_o; + return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); + } - // Intrinsics - SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; - /// isLegalAddressingMode - Return true if the addressing mode represented - /// by AM is legal for this target, for a load/store of the specified type. - /// The type may be VoidTy, in which case only return true if the addressing - /// mode is legal for a load/store of any legal type. - /// TODO: Handle pre/postinc as well. - bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, - Type *Ty, unsigned AS, - Instruction *I = nullptr) const override; - /// Return true if folding a constant offset with the given GlobalAddress - /// is legal. It is frequently not legal in PIC relocation models. - bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; + // Intrinsics + SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; + /// isLegalAddressingMode - Return true if the addressing mode represented + /// by AM is legal for this target, for a load/store of the specified type. + /// The type may be VoidTy, in which case only return true if the addressing + /// mode is legal for a load/store of any legal type. + /// TODO: Handle pre/postinc as well. + bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, + Type *Ty, unsigned AS, + Instruction *I = nullptr) const override; + /// Return true if folding a constant offset with the given GlobalAddress + /// is legal. It is frequently not legal in PIC relocation models. + bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; - bool isFPImmLegal(const APFloat &Imm, EVT VT, - bool ForCodeSize) const override; + bool isFPImmLegal(const APFloat &Imm, EVT VT, + bool ForCodeSize) const override; - /// isLegalICmpImmediate - Return true if the specified immediate is legal - /// icmp immediate, that is the target has icmp instructions which can - /// compare a register against the immediate without having to materialize - /// the immediate into a register. - bool isLegalICmpImmediate(int64_t Imm) const override; + /// isLegalICmpImmediate - Return true if the specified immediate is legal + /// icmp immediate, that is the target has icmp instructions which can + /// compare a register against the immediate without having to materialize + /// the immediate into a register. + bool isLegalICmpImmediate(int64_t Imm) const override; - EVT getOptimalMemOpType(const MemOp &Op, - const AttributeList &FuncAttributes) const override; + EVT getOptimalMemOpType(const MemOp &Op, + const AttributeList &FuncAttributes) const override; - bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, - unsigned AddrSpace, Align Alignment, - MachineMemOperand::Flags Flags, - bool *Fast) const override; + bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, + unsigned AddrSpace, Align Alignment, + MachineMemOperand::Flags Flags, + bool *Fast) const override; - bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, - unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) - const override; + bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, + unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) + const override; - /// Returns relocation base for the given PIC jumptable. - SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) - const override; + /// Returns relocation base for the given PIC jumptable. + SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) + const override; - bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, - EVT NewVT) const override; + bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, + EVT NewVT) const override; - // Handling of atomic RMW instructions. - Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr, - AtomicOrdering Ord) const override; - Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, - Value *Addr, AtomicOrdering Ord) const override; - AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; - bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; - AtomicExpansionKind - shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + // Handling of atomic RMW instructions. + Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr, + AtomicOrdering Ord) const override; + Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, + Value *Addr, AtomicOrdering Ord) const override; + AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; + bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; + AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; - AtomicExpansionKind - shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { - return AtomicExpansionKind::LLSC; - } + AtomicExpansionKind + shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { + return AtomicExpansionKind::LLSC; + } - private: - void initializeHVXLowering(); - void validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl, - unsigned NeedAlign) const; +private: + void initializeHVXLowering(); + unsigned getPreferredHvxVectorAction(MVT VecTy) const; - std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const; + void validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl, + unsigned NeedAlign) const; - bool getBuildVectorConstInts(ArrayRef<SDValue> Values, MVT VecTy, - SelectionDAG &DAG, - MutableArrayRef<ConstantInt*> Consts) const; - SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, - SelectionDAG &DAG) const; - SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, + std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const; + + bool getBuildVectorConstInts(ArrayRef<SDValue> Values, MVT VecTy, + SelectionDAG &DAG, + MutableArrayRef<ConstantInt*> Consts) const; + SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, + SelectionDAG &DAG) const; + SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, + SelectionDAG &DAG) const; + SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl, + MVT ValTy, MVT ResTy, SelectionDAG &DAG) const; + SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, + const SDLoc &dl, MVT ValTy, SelectionDAG &DAG) const; + SDValue expandPredicate(SDValue Vec32, const SDLoc &dl, SelectionDAG &DAG) const; - SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl, - MVT ValTy, MVT ResTy, SelectionDAG &DAG) const; - SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, - const SDLoc &dl, MVT ValTy, SelectionDAG &DAG) const; - SDValue expandPredicate(SDValue Vec32, const SDLoc &dl, + SDValue contractPredicate(SDValue Vec64, const SDLoc &dl, SelectionDAG &DAG) const; - SDValue contractPredicate(SDValue Vec64, const SDLoc &dl, - SelectionDAG &DAG) const; - SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const; + SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const; + SDValue appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG) const; - bool isUndef(SDValue Op) const { - if (Op.isMachineOpcode()) - return Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF; - return Op.getOpcode() == ISD::UNDEF; - } - SDValue getInstr(unsigned MachineOpc, const SDLoc &dl, MVT Ty, - ArrayRef<SDValue> Ops, SelectionDAG &DAG) const { - SDNode *N = DAG.getMachineNode(MachineOpc, dl, Ty, Ops); - return SDValue(N, 0); - } - SDValue getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) const; + bool isUndef(SDValue Op) const { + if (Op.isMachineOpcode()) + return Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF; + return Op.getOpcode() == ISD::UNDEF; + } + SDValue getInstr(unsigned MachineOpc, const SDLoc &dl, MVT Ty, + ArrayRef<SDValue> Ops, SelectionDAG &DAG) const { + SDNode *N = DAG.getMachineNode(MachineOpc, dl, Ty, Ops); + return SDValue(N, 0); + } + SDValue getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) const; - using VectorPair = std::pair<SDValue, SDValue>; - using TypePair = std::pair<MVT, MVT>; + using VectorPair = std::pair<SDValue, SDValue>; + using TypePair = std::pair<MVT, MVT>; - SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, - const SDLoc &dl, SelectionDAG &DAG) const; + SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, + const SDLoc &dl, SelectionDAG &DAG) const; - MVT ty(SDValue Op) const { - return Op.getValueType().getSimpleVT(); - } - TypePair ty(const VectorPair &Ops) const { - return { Ops.first.getValueType().getSimpleVT(), - Ops.second.getValueType().getSimpleVT() }; - } - MVT tyScalar(MVT Ty) const { - if (!Ty.isVector()) - return Ty; - return MVT::getIntegerVT(Ty.getSizeInBits()); - } - MVT tyVector(MVT Ty, MVT ElemTy) const { - if (Ty.isVector() && Ty.getVectorElementType() == ElemTy) - return Ty; - unsigned TyWidth = Ty.getSizeInBits(); - unsigned ElemWidth = ElemTy.getSizeInBits(); - assert((TyWidth % ElemWidth) == 0); - return MVT::getVectorVT(ElemTy, TyWidth/ElemWidth); - } + MVT ty(SDValue Op) const { + return Op.getValueType().getSimpleVT(); + } + TypePair ty(const VectorPair &Ops) const { + return { Ops.first.getValueType().getSimpleVT(), + Ops.second.getValueType().getSimpleVT() }; + } + MVT tyScalar(MVT Ty) const { + if (!Ty.isVector()) + return Ty; + return MVT::getIntegerVT(Ty.getSizeInBits()); + } + MVT tyVector(MVT Ty, MVT ElemTy) const { + if (Ty.isVector() && Ty.getVectorElementType() == ElemTy) + return Ty; + unsigned TyWidth = Ty.getSizeInBits(); + unsigned ElemWidth = ElemTy.getSizeInBits(); + assert((TyWidth % ElemWidth) == 0); + return MVT::getVectorVT(ElemTy, TyWidth/ElemWidth); + } - MVT typeJoin(const TypePair &Tys) const; - TypePair typeSplit(MVT Ty) const; - MVT typeExtElem(MVT VecTy, unsigned Factor) const; - MVT typeTruncElem(MVT VecTy, unsigned Factor) const; + MVT typeJoin(const TypePair &Tys) const; + TypePair typeSplit(MVT Ty) const; + MVT typeExtElem(MVT VecTy, unsigned Factor) const; + MVT typeTruncElem(MVT VecTy, unsigned Factor) const; - SDValue opJoin(const VectorPair &Ops, const SDLoc &dl, - SelectionDAG &DAG) const; - VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const; - SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const; + SDValue opJoin(const VectorPair &Ops, const SDLoc &dl, + SelectionDAG &DAG) const; + VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const; + SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const; - bool allowsHvxMemoryAccess(MVT VecTy, MachineMemOperand::Flags Flags, - bool *Fast) const; - bool allowsHvxMisalignedMemoryAccesses(MVT VecTy, - MachineMemOperand::Flags Flags, - bool *Fast) const; + bool allowsHvxMemoryAccess(MVT VecTy, MachineMemOperand::Flags Flags, + bool *Fast) const; + bool allowsHvxMisalignedMemoryAccesses(MVT VecTy, + MachineMemOperand::Flags Flags, + bool *Fast) const; - bool isHvxSingleTy(MVT Ty) const; - bool isHvxPairTy(MVT Ty) const; - bool isHvxBoolTy(MVT Ty) const; - SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy, - SelectionDAG &DAG) const; - SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const; - SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1, - ArrayRef<int> Mask, SelectionDAG &DAG) const; + bool isHvxSingleTy(MVT Ty) const; + bool isHvxPairTy(MVT Ty) const; + bool isHvxBoolTy(MVT Ty) const; + SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy, + SelectionDAG &DAG) const; + SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const; + SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1, + ArrayRef<int> Mask, SelectionDAG &DAG) const; - SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl, - MVT VecTy, SelectionDAG &DAG) const; - SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl, - MVT VecTy, SelectionDAG &DAG) const; - SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl, - unsigned BitBytes, bool ZeroFill, - SelectionDAG &DAG) const; - SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, + SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl, + MVT VecTy, SelectionDAG &DAG) const; + SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl, + MVT VecTy, SelectionDAG &DAG) const; + SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl, + unsigned BitBytes, bool ZeroFill, + SelectionDAG &DAG) const; + SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, + MVT ResTy, SelectionDAG &DAG) const; + SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, + MVT ResTy, SelectionDAG &DAG) const; + SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV, + const SDLoc &dl, SelectionDAG &DAG) const; + SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV, + const SDLoc &dl, SelectionDAG &DAG) const; + SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const; - SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, + SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const; - SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV, + SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const; - SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV, + SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const; - SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, - MVT ResTy, SelectionDAG &DAG) const; - SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, - MVT ResTy, SelectionDAG &DAG) const; - SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV, - const SDLoc &dl, SelectionDAG &DAG) const; - SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV, - const SDLoc &dl, SelectionDAG &DAG) const; - SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy, - bool ZeroExt, SelectionDAG &DAG) const; - SDValue compressHvxPred(SDValue VecQ, const SDLoc &dl, MVT ResTy, - SelectionDAG &DAG) const; + SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy, + bool ZeroExt, SelectionDAG &DAG) const; + SDValue compressHvxPred(SDValue VecQ, const SDLoc &dl, MVT ResTy, + SelectionDAG &DAG) const; - SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerHvxStore(SDValue Op, SelectionDAG &DAG) const; - SDValue HvxVecPredBitcastComputation(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const; - SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const; - SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const; + SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const; + SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const; + SDValue WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const; + SDValue WidenHvxStore(SDValue Op, SelectionDAG &DAG) const; + SDValue WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const; + SDValue WidenHvxExtend(SDValue Op, SelectionDAG &DAG) const; + SDValue WidenHvxTruncate(SDValue Op, SelectionDAG &DAG) const; - std::pair<const TargetRegisterClass*, uint8_t> - findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) - const override; + std::pair<const TargetRegisterClass*, uint8_t> + findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) + const override; - bool isHvxOperation(SDValue Op) const; - bool isHvxOperation(SDNode *N) const; - SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const; - void LowerHvxOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) const; - void ReplaceHvxNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) const; - SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; - }; + bool shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const; + bool isHvxOperation(SDNode *N, SelectionDAG &DAG) const; + SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const; + void LowerHvxOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, + SelectionDAG &DAG) const; + void ReplaceHvxNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, + SelectionDAG &DAG) const; + SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; +}; } // end namespace llvm diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp index 7cda915fffe9..29b75814da6e 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp @@ -14,6 +14,10 @@ using namespace llvm; +static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen", + cl::Hidden, cl::init(16), + cl::desc("Lower threshold (in bytes) for widening to HVX vectors")); + static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; @@ -87,17 +91,28 @@ HexagonTargetLowering::initializeHVXLowering() { setOperationAction(ISD::XOR, T, Legal); setOperationAction(ISD::ADD, T, Legal); setOperationAction(ISD::SUB, T, Legal); + setOperationAction(ISD::MUL, T, Legal); setOperationAction(ISD::CTPOP, T, Legal); setOperationAction(ISD::CTLZ, T, Legal); + setOperationAction(ISD::SELECT, T, Legal); + setOperationAction(ISD::SPLAT_VECTOR, T, Legal); if (T != ByteV) { setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); setOperationAction(ISD::BSWAP, T, Legal); } + setOperationAction(ISD::SMIN, T, Legal); + setOperationAction(ISD::SMAX, T, Legal); + if (T.getScalarType() != MVT::i32) { + setOperationAction(ISD::UMIN, T, Legal); + setOperationAction(ISD::UMAX, T, Legal); + } + setOperationAction(ISD::CTTZ, T, Custom); setOperationAction(ISD::LOAD, T, Custom); - setOperationAction(ISD::MUL, T, Custom); + setOperationAction(ISD::MLOAD, T, Custom); + setOperationAction(ISD::MSTORE, T, Custom); setOperationAction(ISD::MULHS, T, Custom); setOperationAction(ISD::MULHU, T, Custom); setOperationAction(ISD::BUILD_VECTOR, T, Custom); @@ -147,9 +162,12 @@ HexagonTargetLowering::initializeHVXLowering() { setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); + setOperationAction(ISD::SPLAT_VECTOR, T, Custom); setOperationAction(ISD::LOAD, T, Custom); setOperationAction(ISD::STORE, T, Custom); + setOperationAction(ISD::MLOAD, T, Custom); + setOperationAction(ISD::MSTORE, T, Custom); setOperationAction(ISD::CTLZ, T, Custom); setOperationAction(ISD::CTTZ, T, Custom); setOperationAction(ISD::CTPOP, T, Custom); @@ -172,6 +190,13 @@ HexagonTargetLowering::initializeHVXLowering() { // Promote all shuffles to operate on vectors of bytes. setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); } + + setOperationAction(ISD::SMIN, T, Custom); + setOperationAction(ISD::SMAX, T, Custom); + if (T.getScalarType() != MVT::i32) { + setOperationAction(ISD::UMIN, T, Custom); + setOperationAction(ISD::UMAX, T, Custom); + } } // Boolean vectors. @@ -188,6 +213,9 @@ HexagonTargetLowering::initializeHVXLowering() { setOperationAction(ISD::AND, BoolW, Custom); setOperationAction(ISD::OR, BoolW, Custom); setOperationAction(ISD::XOR, BoolW, Custom); + // Masked load/store takes a mask that may need splitting. + setOperationAction(ISD::MLOAD, BoolW, Custom); + setOperationAction(ISD::MSTORE, BoolW, Custom); } for (MVT T : LegalV) { @@ -198,6 +226,7 @@ HexagonTargetLowering::initializeHVXLowering() { setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); + setOperationAction(ISD::SELECT, BoolV, Custom); setOperationAction(ISD::AND, BoolV, Legal); setOperationAction(ISD::OR, BoolV, Legal); setOperationAction(ISD::XOR, BoolV, Legal); @@ -211,16 +240,82 @@ HexagonTargetLowering::initializeHVXLowering() { setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal); } + // Handle store widening for short vectors. + unsigned HwLen = Subtarget.getVectorLength(); + for (MVT ElemTy : Subtarget.getHVXElementTypes()) { + if (ElemTy == MVT::i1) + continue; + int ElemWidth = ElemTy.getFixedSizeInBits(); + int MaxElems = (8*HwLen) / ElemWidth; + for (int N = 2; N < MaxElems; N *= 2) { + MVT VecTy = MVT::getVectorVT(ElemTy, N); + auto Action = getPreferredVectorAction(VecTy); + if (Action == TargetLoweringBase::TypeWidenVector) { + setOperationAction(ISD::LOAD, VecTy, Custom); + setOperationAction(ISD::STORE, VecTy, Custom); + setOperationAction(ISD::SETCC, VecTy, Custom); + setOperationAction(ISD::TRUNCATE, VecTy, Custom); + setOperationAction(ISD::ANY_EXTEND, VecTy, Custom); + setOperationAction(ISD::SIGN_EXTEND, VecTy, Custom); + setOperationAction(ISD::ZERO_EXTEND, VecTy, Custom); + + MVT BoolTy = MVT::getVectorVT(MVT::i1, N); + if (!isTypeLegal(BoolTy)) + setOperationAction(ISD::SETCC, BoolTy, Custom); + } + } + } + + setTargetDAGCombine(ISD::SPLAT_VECTOR); setTargetDAGCombine(ISD::VSELECT); } +unsigned +HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const { + MVT ElemTy = VecTy.getVectorElementType(); + unsigned VecLen = VecTy.getVectorNumElements(); + unsigned HwLen = Subtarget.getVectorLength(); + + // Split vectors of i1 that exceed byte vector length. + if (ElemTy == MVT::i1 && VecLen > HwLen) + return TargetLoweringBase::TypeSplitVector; + + ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes(); + // For shorter vectors of i1, widen them if any of the corresponding + // vectors of integers needs to be widened. + if (ElemTy == MVT::i1) { + for (MVT T : Tys) { + assert(T != MVT::i1); + auto A = getPreferredHvxVectorAction(MVT::getVectorVT(T, VecLen)); + if (A != ~0u) + return A; + } + return ~0u; + } + + // If the size of VecTy is at least half of the vector length, + // widen the vector. Note: the threshold was not selected in + // any scientific way. + if (llvm::is_contained(Tys, ElemTy)) { + unsigned VecWidth = VecTy.getSizeInBits(); + bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0; + if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth) + return TargetLoweringBase::TypeWidenVector; + unsigned HwWidth = 8*HwLen; + if (VecWidth >= HwWidth/2 && VecWidth < HwWidth) + return TargetLoweringBase::TypeWidenVector; + } + + // Defer to default. + return ~0u; +} + SDValue HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, const SDLoc &dl, SelectionDAG &DAG) const { SmallVector<SDValue,4> IntOps; IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32)); - for (const SDValue &Op : Ops) - IntOps.push_back(Op); + append_range(IntOps, Ops); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps); } @@ -432,7 +527,9 @@ HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values, auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode()); if (IdxN && IdxN->isNullValue()) return getZero(dl, VecTy, DAG); - return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV); + MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4); + SDValue S = DAG.getNode(ISD::SPLAT_VECTOR, dl, WordTy, SplatV); + return DAG.getBitcast(VecTy, S); } // Delay recognizing constant vectors until here, so that we can generate @@ -571,6 +668,9 @@ HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl, if (!ZeroFill) return S; // Fill the bytes beyond BlockLen with 0s. + // V6_pred_scalar2 cannot fill the entire predicate, so it only works + // when BlockLen < HwLen. + assert(BlockLen < HwLen && "vsetq(v1) prerequisite"); MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); @@ -1034,6 +1134,7 @@ HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV, // ByteVec is the target vector VecV rotated in such a way that the // subvector should be inserted at index 0. Generate a predicate mask // and use vmux to do the insertion. + assert(BlockLen < HwLen && "vsetq(v1) prerequisite"); MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); @@ -1058,7 +1159,7 @@ HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl, return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV); assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements()); - SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy, + SDValue True = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, DAG.getConstant(1, dl, MVT::i32)); SDValue False = getZero(dl, ResTy, DAG); return DAG.getSelect(dl, ResTy, VecV, True, False); @@ -1180,12 +1281,19 @@ HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) continue; } // A few less complicated cases. - if (V.getOpcode() == ISD::Constant) - Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy); - else if (V.isUndef()) - Elems[i] = DAG.getUNDEF(NTy); - else - llvm_unreachable("Unexpected vector element"); + switch (V.getOpcode()) { + case ISD::Constant: + Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy); + break; + case ISD::UNDEF: + Elems[i] = DAG.getUNDEF(NTy); + break; + case ISD::TRUNCATE: + Elems[i] = V.getOperand(0); + break; + default: + llvm_unreachable("Unexpected vector element"); + } } } return DAG.getBuildVector(VecTy, dl, Elems); @@ -1346,19 +1454,14 @@ HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const { // Calculate the vectors of 1 and bitwidth(x). MVT ElemTy = ty(InpV).getVectorElementType(); unsigned ElemWidth = ElemTy.getSizeInBits(); - // Using uint64_t because a shift by 32 can happen. - uint64_t Splat1 = 0, SplatW = 0; - assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32); - for (unsigned i = 0; i != 32/ElemWidth; ++i) { - Splat1 = (Splat1 << ElemWidth) | 1; - SplatW = (SplatW << ElemWidth) | ElemWidth; - } - SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, - DAG.getConstant(uint32_t(Splat1), dl, MVT::i32)); - SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, - DAG.getConstant(uint32_t(SplatW), dl, MVT::i32)); - SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, + + SDValue Vec1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, + DAG.getConstant(1, dl, MVT::i32)); + SDValue VecW = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, + DAG.getConstant(ElemWidth, dl, MVT::i32)); + SDValue VecN1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, DAG.getConstant(-1, dl, MVT::i32)); + // Do not use DAG.getNOT, because that would create BUILD_VECTOR with // a BITCAST. Here we can skip the BITCAST (so we don't have to handle // it separately in custom combine or selection). @@ -1370,60 +1473,6 @@ HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const { } SDValue -HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const { - MVT ResTy = ty(Op); - assert(ResTy.isVector() && isHvxSingleTy(ResTy)); - const SDLoc &dl(Op); - SmallVector<int,256> ShuffMask; - - MVT ElemTy = ResTy.getVectorElementType(); - unsigned VecLen = ResTy.getVectorNumElements(); - SDValue Vs = Op.getOperand(0); - SDValue Vt = Op.getOperand(1); - - switch (ElemTy.SimpleTy) { - case MVT::i8: { - // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), - // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, - // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). - MVT ExtTy = typeExtElem(ResTy, 2); - unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv - : Hexagon::V6_vmpyhv; - SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG); - - // Discard high halves of the resulting values, collect the low halves. - for (unsigned I = 0; I < VecLen; I += 2) { - ShuffMask.push_back(I); // Pick even element. - ShuffMask.push_back(I+VecLen); // Pick odd element. - } - VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG); - SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG); - return DAG.getBitcast(ResTy, BS); - } - case MVT::i16: - // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode. - // (There is also V6_vmpyhv, which behaves in an analogous way to - // V6_vmpybv.) - return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG); - case MVT::i32: { - // Use the following sequence for signed word multiply: - // T0 = V6_vmpyiowh Vs, Vt - // T1 = V6_vaslw T0, 16 - // T2 = V6_vmpyiewuh_acc T1, Vs, Vt - SDValue S16 = DAG.getConstant(16, dl, MVT::i32); - SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG); - SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG); - SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy, - {T1, Vs, Vt}, DAG); - return T2; - } - default: - break; - } - return SDValue(); -} - -SDValue HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { MVT ResTy = ty(Op); assert(ResTy.isVector()); @@ -1462,7 +1511,7 @@ HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { assert(ElemTy == MVT::i32); SDValue S16 = DAG.getConstant(16, dl, MVT::i32); - if (IsSigned) { + auto MulHS_V60 = [&](SDValue Vs, SDValue Vt) { // mulhs(Vs,Vt) = // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32 // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16 @@ -1489,6 +1538,20 @@ HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { // Add: SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2}); return T3; + }; + + auto MulHS_V62 = [&](SDValue Vs, SDValue Vt) { + MVT PairTy = typeJoin({ResTy, ResTy}); + SDValue T0 = getInstr(Hexagon::V6_vmpyewuh_64, dl, PairTy, {Vs, Vt}, DAG); + SDValue T1 = getInstr(Hexagon::V6_vmpyowh_64_acc, dl, PairTy, + {T0, Vs, Vt}, DAG); + return opSplit(T1, dl, DAG).second; + }; + + if (IsSigned) { + if (Subtarget.useHVXV62Ops()) + return MulHS_V62(Vs, Vt); + return MulHS_V60(Vs, Vt); } // Unsigned mulhw. (Would expansion using signed mulhw be better?) @@ -1585,6 +1648,26 @@ HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const { } SDValue +HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const { + MVT ResTy = ty(Op); + if (ResTy.getVectorElementType() != MVT::i1) + return Op; + + const SDLoc &dl(Op); + unsigned HwLen = Subtarget.getVectorLength(); + unsigned VecLen = ResTy.getVectorNumElements(); + assert(HwLen % VecLen == 0); + unsigned ElemSize = HwLen / VecLen; + + MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(ElemSize * 8), VecLen); + SDValue S = + DAG.getNode(ISD::SELECT, dl, VecTy, Op.getOperand(0), + DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(1)), + DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(2))); + return DAG.getNode(HexagonISD::V2Q, dl, ResTy, S); +} + +SDValue HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { if (SDValue S = getVectorShiftByInt(Op, DAG)) return S; @@ -1593,7 +1676,7 @@ HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { SDValue HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const { - const SDLoc &dl(Op); + const SDLoc &dl(Op); MVT ResTy = ty(Op); unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); @@ -1614,6 +1697,76 @@ HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const { } SDValue +HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + unsigned HwLen = Subtarget.getVectorLength(); + MachineFunction &MF = DAG.getMachineFunction(); + auto *MaskN = cast<MaskedLoadStoreSDNode>(Op.getNode()); + SDValue Mask = MaskN->getMask(); + SDValue Chain = MaskN->getChain(); + SDValue Base = MaskN->getBasePtr(); + auto *MemOp = MF.getMachineMemOperand(MaskN->getMemOperand(), 0, HwLen); + + unsigned Opc = Op->getOpcode(); + assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE); + + if (Opc == ISD::MLOAD) { + MVT ValTy = ty(Op); + SDValue Load = DAG.getLoad(ValTy, dl, Chain, Base, MemOp); + SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru(); + if (isUndef(Thru)) + return Load; + SDValue VSel = DAG.getNode(ISD::VSELECT, dl, ValTy, Mask, Load, Thru); + return DAG.getMergeValues({VSel, Load.getValue(1)}, dl); + } + + // MSTORE + // HVX only has aligned masked stores. + + // TODO: Fold negations of the mask into the store. + unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai; + SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue(); + SDValue Offset0 = DAG.getTargetConstant(0, dl, ty(Base)); + + if (MaskN->getAlign().value() % HwLen == 0) { + SDValue Store = getInstr(StoreOpc, dl, MVT::Other, + {Mask, Base, Offset0, Value, Chain}, DAG); + DAG.setNodeMemRefs(cast<MachineSDNode>(Store.getNode()), {MemOp}); + return Store; + } + + // Unaligned case. + auto StoreAlign = [&](SDValue V, SDValue A) { + SDValue Z = getZero(dl, ty(V), DAG); + // TODO: use funnel shifts? + // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the + // upper half. + SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG); + SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG); + return std::make_pair(LoV, HiV); + }; + + MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); + MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); + SDValue MaskV = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Mask); + VectorPair Tmp = StoreAlign(MaskV, Base); + VectorPair MaskU = {DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.first), + DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.second)}; + VectorPair ValueU = StoreAlign(Value, Base); + + SDValue Offset1 = DAG.getTargetConstant(HwLen, dl, MVT::i32); + SDValue StoreLo = + getInstr(StoreOpc, dl, MVT::Other, + {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG); + SDValue StoreHi = + getInstr(StoreOpc, dl, MVT::Other, + {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG); + DAG.setNodeMemRefs(cast<MachineSDNode>(StoreLo.getNode()), {MemOp}); + DAG.setNodeMemRefs(cast<MachineSDNode>(StoreHi.getNode()), {MemOp}); + return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, {StoreLo, StoreHi}); +} + +SDValue HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const { assert(!Op.isMachineOpcode()); SmallVector<SDValue,2> OpsL, OpsH; @@ -1648,45 +1801,252 @@ HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const { SDValue HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const { - LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode()); - assert(BN->isUnindexed()); - MVT MemTy = BN->getMemoryVT().getSimpleVT(); + auto *MemN = cast<MemSDNode>(Op.getNode()); + + MVT MemTy = MemN->getMemoryVT().getSimpleVT(); if (!isHvxPairTy(MemTy)) return Op; const SDLoc &dl(Op); unsigned HwLen = Subtarget.getVectorLength(); MVT SingleTy = typeSplit(MemTy).first; - SDValue Chain = BN->getChain(); - SDValue Base0 = BN->getBasePtr(); - SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl); + SDValue Chain = MemN->getChain(); + SDValue Base0 = MemN->getBasePtr(); + SDValue Base1 = DAG.getMemBasePlusOffset(Base0, TypeSize::Fixed(HwLen), dl); MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr; - if (MachineMemOperand *MMO = BN->getMemOperand()) { + if (MachineMemOperand *MMO = MemN->getMemOperand()) { MachineFunction &MF = DAG.getMachineFunction(); MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen); MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen); } - unsigned MemOpc = BN->getOpcode(); - SDValue NewOp; + unsigned MemOpc = MemN->getOpcode(); if (MemOpc == ISD::LOAD) { + assert(cast<LoadSDNode>(Op)->isUnindexed()); SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0); SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1); - NewOp = DAG.getMergeValues( - { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1), - DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - Load0.getValue(1), Load1.getValue(1)) }, dl); - } else { - assert(MemOpc == ISD::STORE); + return DAG.getMergeValues( + { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1), + DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + Load0.getValue(1), Load1.getValue(1)) }, dl); + } + if (MemOpc == ISD::STORE) { + assert(cast<StoreSDNode>(Op)->isUnindexed()); VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG); SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0); SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1); - NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1); + return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1); + } + + assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE); + + auto MaskN = cast<MaskedLoadStoreSDNode>(Op); + assert(MaskN->isUnindexed()); + VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG); + SDValue Offset = DAG.getUNDEF(MVT::i32); + + if (MemOpc == ISD::MLOAD) { + VectorPair Thru = + opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG); + SDValue MLoad0 = + DAG.getMaskedLoad(SingleTy, dl, Chain, Base0, Offset, Masks.first, + Thru.first, SingleTy, MOp0, ISD::UNINDEXED, + ISD::NON_EXTLOAD, false); + SDValue MLoad1 = + DAG.getMaskedLoad(SingleTy, dl, Chain, Base1, Offset, Masks.second, + Thru.second, SingleTy, MOp1, ISD::UNINDEXED, + ISD::NON_EXTLOAD, false); + return DAG.getMergeValues( + { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, MLoad0, MLoad1), + DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + MLoad0.getValue(1), MLoad1.getValue(1)) }, dl); + } + if (MemOpc == ISD::MSTORE) { + VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG); + SDValue MStore0 = DAG.getMaskedStore(Chain, dl, Vals.first, Base0, Offset, + Masks.first, SingleTy, MOp0, + ISD::UNINDEXED, false, false); + SDValue MStore1 = DAG.getMaskedStore(Chain, dl, Vals.second, Base1, Offset, + Masks.second, SingleTy, MOp1, + ISD::UNINDEXED, false, false); + return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MStore0, MStore1); + } + + std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG); + llvm_unreachable(Name.c_str()); +} + +SDValue +HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + auto *LoadN = cast<LoadSDNode>(Op.getNode()); + assert(LoadN->isUnindexed() && "Not widening indexed loads yet"); + assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 && + "Not widening loads of i1 yet"); + + SDValue Chain = LoadN->getChain(); + SDValue Base = LoadN->getBasePtr(); + SDValue Offset = DAG.getUNDEF(MVT::i32); + + MVT ResTy = ty(Op); + unsigned HwLen = Subtarget.getVectorLength(); + unsigned ResLen = ResTy.getStoreSize(); + assert(ResLen < HwLen && "vsetq(v1) prerequisite"); + + MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); + SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, + {DAG.getConstant(ResLen, dl, MVT::i32)}, DAG); + + MVT LoadTy = MVT::getVectorVT(MVT::i8, HwLen); + MachineFunction &MF = DAG.getMachineFunction(); + auto *MemOp = MF.getMachineMemOperand(LoadN->getMemOperand(), 0, HwLen); + + SDValue Load = DAG.getMaskedLoad(LoadTy, dl, Chain, Base, Offset, Mask, + DAG.getUNDEF(LoadTy), LoadTy, MemOp, + ISD::UNINDEXED, ISD::NON_EXTLOAD, false); + SDValue Value = opCastElem(Load, ResTy.getVectorElementType(), DAG); + return DAG.getMergeValues({Value, Chain}, dl); +} + +SDValue +HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + auto *StoreN = cast<StoreSDNode>(Op.getNode()); + assert(StoreN->isUnindexed() && "Not widening indexed stores yet"); + assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 && + "Not widening stores of i1 yet"); + + SDValue Chain = StoreN->getChain(); + SDValue Base = StoreN->getBasePtr(); + SDValue Offset = DAG.getUNDEF(MVT::i32); + + SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG); + MVT ValueTy = ty(Value); + unsigned ValueLen = ValueTy.getVectorNumElements(); + unsigned HwLen = Subtarget.getVectorLength(); + assert(isPowerOf2_32(ValueLen)); + + for (unsigned Len = ValueLen; Len < HwLen; ) { + Value = opJoin({DAG.getUNDEF(ty(Value)), Value}, dl, DAG); + Len = ty(Value).getVectorNumElements(); // This is Len *= 2 } + assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia - return NewOp; + assert(ValueLen < HwLen && "vsetq(v1) prerequisite"); + MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); + SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, + {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG); + MachineFunction &MF = DAG.getMachineFunction(); + auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen); + return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, Mask, ty(Value), + MemOp, ISD::UNINDEXED, false, false); +} + +SDValue +HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1); + MVT ElemTy = ty(Op0).getVectorElementType(); + unsigned HwLen = Subtarget.getVectorLength(); + + unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits(); + assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen); + MVT WideOpTy = MVT::getVectorVT(ElemTy, WideOpLen); + + SDValue WideOp0 = appendUndef(Op0, WideOpTy, DAG); + SDValue WideOp1 = appendUndef(Op1, WideOpTy, DAG); + EVT ResTy = + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), WideOpTy); + SDValue SetCC = DAG.getNode(ISD::SETCC, dl, ResTy, + {WideOp0, WideOp1, Op.getOperand(2)}); + + EVT RetTy = getTypeToTransformTo(*DAG.getContext(), ty(Op)); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RetTy, + {SetCC, getZero(dl, MVT::i32, DAG)}); +} + +SDValue +HexagonTargetLowering::WidenHvxExtend(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + unsigned HwWidth = 8*Subtarget.getVectorLength(); + + SDValue Op0 = Op.getOperand(0); + MVT ResTy = ty(Op); + MVT OpTy = ty(Op0); + if (!Subtarget.isHVXElementType(OpTy) || !Subtarget.isHVXElementType(ResTy)) + return SDValue(); + + // .-res, op-> ScalarVec Illegal HVX + // Scalar ok - - + // Illegal widen(insert) widen - + // HVX - widen ok + + auto getFactor = [HwWidth](MVT Ty) { + unsigned Width = Ty.getSizeInBits(); + return HwWidth > Width ? HwWidth / Width : 1; + }; + + auto getWideTy = [getFactor](MVT Ty) { + unsigned WideLen = Ty.getVectorNumElements() * getFactor(Ty); + return MVT::getVectorVT(Ty.getVectorElementType(), WideLen); + }; + + unsigned Opcode = Op.getOpcode() == ISD::SIGN_EXTEND ? HexagonISD::VUNPACK + : HexagonISD::VUNPACKU; + SDValue WideOp = appendUndef(Op0, getWideTy(OpTy), DAG); + SDValue WideRes = DAG.getNode(Opcode, dl, getWideTy(ResTy), WideOp); + return WideRes; +} + +SDValue +HexagonTargetLowering::WidenHvxTruncate(SDValue Op, SelectionDAG &DAG) const { + const SDLoc &dl(Op); + unsigned HwWidth = 8*Subtarget.getVectorLength(); + + SDValue Op0 = Op.getOperand(0); + MVT ResTy = ty(Op); + MVT OpTy = ty(Op0); + if (!Subtarget.isHVXElementType(OpTy) || !Subtarget.isHVXElementType(ResTy)) + return SDValue(); + + // .-res, op-> ScalarVec Illegal HVX + // Scalar ok extract(widen) - + // Illegal - widen widen + // HVX - - ok + + auto getFactor = [HwWidth](MVT Ty) { + unsigned Width = Ty.getSizeInBits(); + assert(HwWidth % Width == 0); + return HwWidth / Width; + }; + + auto getWideTy = [getFactor](MVT Ty) { + unsigned WideLen = Ty.getVectorNumElements() * getFactor(Ty); + return MVT::getVectorVT(Ty.getVectorElementType(), WideLen); + }; + + if (Subtarget.isHVXVectorType(OpTy)) + return DAG.getNode(HexagonISD::VPACKL, dl, getWideTy(ResTy), Op0); + + assert(!isTypeLegal(OpTy) && "HVX-widening a truncate of scalar?"); + + SDValue WideOp = appendUndef(Op0, getWideTy(OpTy), DAG); + SDValue WideRes = DAG.getNode(HexagonISD::VPACKL, dl, getWideTy(ResTy), + WideOp); + // If the original result wasn't legal and was supposed to be widened, + // we're done. + if (shouldWidenToHvx(ResTy, DAG)) + return WideRes; + + // The original result type wasn't meant to be widened to HVX, so + // leave it as it is. Standard legalization should be able to deal + // with it (since now it's a result of a target-idendependent ISD + // node). + assert(ResTy.isVector()); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResTy, + {WideRes, getZero(dl, MVT::i32, DAG)}); } SDValue @@ -1703,6 +2063,8 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { break; case ISD::LOAD: case ISD::STORE: + case ISD::MLOAD: + case ISD::MSTORE: return SplitHvxMemOp(Op, DAG); case ISD::CTPOP: case ISD::CTLZ: @@ -1716,11 +2078,16 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::SRA: case ISD::SHL: case ISD::SRL: + case ISD::SMIN: + case ISD::SMAX: + case ISD::UMIN: + case ISD::UMAX: case ISD::SETCC: case ISD::VSELECT: case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: case ISD::SIGN_EXTEND_INREG: + case ISD::SPLAT_VECTOR: return SplitHvxPairOp(Op, DAG); } } @@ -1739,16 +2106,18 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG); case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG); case ISD::CTTZ: return LowerHvxCttz(Op, DAG); + case ISD::SELECT: return LowerHvxSelect(Op, DAG); case ISD::SRA: case ISD::SHL: case ISD::SRL: return LowerHvxShift(Op, DAG); - case ISD::MUL: return LowerHvxMul(Op, DAG); case ISD::MULHS: case ISD::MULHU: return LowerHvxMulh(Op, DAG); case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG); case ISD::SETCC: case ISD::INTRINSIC_VOID: return Op; case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG); + case ISD::MLOAD: + case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG); // Unaligned loads will be handled by the default lowering. case ISD::LOAD: return SDValue(); } @@ -1761,13 +2130,91 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { void HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { + unsigned Opc = N->getOpcode(); + SDValue Op(N, 0); + + switch (Opc) { + case ISD::ANY_EXTEND: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + if (shouldWidenToHvx(ty(Op.getOperand(0)), DAG)) { + if (SDValue T = WidenHvxExtend(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::SETCC: + if (shouldWidenToHvx(ty(Op.getOperand(0)), DAG)) { + if (SDValue T = WidenHvxSetCC(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::TRUNCATE: + if (shouldWidenToHvx(ty(Op.getOperand(0)), DAG)) { + if (SDValue T = WidenHvxTruncate(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::STORE: { + if (shouldWidenToHvx(ty(cast<StoreSDNode>(N)->getValue()), DAG)) { + SDValue Store = WidenHvxStore(Op, DAG); + Results.push_back(Store); + } + break; + } + case ISD::MLOAD: + if (isHvxPairTy(ty(Op))) { + SDValue S = SplitHvxMemOp(Op, DAG); + assert(S->getOpcode() == ISD::MERGE_VALUES); + Results.push_back(S.getOperand(0)); + Results.push_back(S.getOperand(1)); + } + break; + case ISD::MSTORE: + if (isHvxPairTy(ty(Op->getOperand(1)))) { // Stored value + SDValue S = SplitHvxMemOp(Op, DAG); + Results.push_back(S); + } + break; + default: + break; + } } void HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { unsigned Opc = N->getOpcode(); + SDValue Op(N, 0); switch (Opc) { + case ISD::ANY_EXTEND: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + if (shouldWidenToHvx(ty(Op), DAG)) { + if (SDValue T = WidenHvxExtend(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::SETCC: + if (shouldWidenToHvx(ty(Op), DAG)) { + if (SDValue T = WidenHvxSetCC(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::TRUNCATE: + if (shouldWidenToHvx(ty(Op), DAG)) { + if (SDValue T = WidenHvxTruncate(Op, DAG)) + Results.push_back(T); + } + break; + case ISD::LOAD: { + if (shouldWidenToHvx(ty(Op), DAG)) { + SDValue Load = WidenHvxLoad(Op, DAG); + assert(Load->getOpcode() == ISD::MERGE_VALUES); + Results.push_back(Load.getOperand(0)); + Results.push_back(Load.getOperand(1)); + } + break; + } case ISD::BITCAST: if (isHvxBoolTy(ty(N->getOperand(0)))) { SDValue Op(N, 0); @@ -1784,44 +2231,95 @@ SDValue HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { const SDLoc &dl(N); + SelectionDAG &DAG = DCI.DAG; SDValue Op(N, 0); - unsigned Opc = Op.getOpcode(); - if (Opc == ISD::VSELECT) { - // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0) - SDValue Cond = Op.getOperand(0); - if (Cond->getOpcode() == ISD::XOR) { - SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); - if (C1->getOpcode() == HexagonISD::QTRUE) { - SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, - Op.getOperand(2), Op.getOperand(1)); - return VSel; + if (DCI.isBeforeLegalizeOps()) + return SDValue(); + + SmallVector<SDValue, 4> Ops(N->ops().begin(), N->ops().end()); + + switch (Opc) { + case ISD::VSELECT: { + // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0) + SDValue Cond = Ops[0]; + if (Cond->getOpcode() == ISD::XOR) { + SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); + if (C1->getOpcode() == HexagonISD::QTRUE) + return DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, Ops[2], Ops[1]); + } + break; + } + case HexagonISD::V2Q: + if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) { + if (const auto *C = dyn_cast<ConstantSDNode>(Ops[0].getOperand(0))) + return C->isNullValue() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op)) + : DAG.getNode(HexagonISD::QTRUE, dl, ty(Op)); + } + break; + case HexagonISD::Q2V: + if (Ops[0].getOpcode() == HexagonISD::QTRUE) + return DAG.getNode(ISD::SPLAT_VECTOR, dl, ty(Op), + DAG.getConstant(-1, dl, MVT::i32)); + if (Ops[0].getOpcode() == HexagonISD::QFALSE) + return getZero(dl, ty(Op), DAG); + break; + case HexagonISD::VINSERTW0: + if (isUndef(Ops[1])) + return Ops[0];; + break; + case HexagonISD::VROR: { + if (Ops[0].getOpcode() == HexagonISD::VROR) { + SDValue Vec = Ops[0].getOperand(0); + SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(1); + SDValue Rot = DAG.getNode(ISD::ADD, dl, ty(Rot0), {Rot0, Rot1}); + return DAG.getNode(HexagonISD::VROR, dl, ty(Op), {Vec, Rot}); } + break; } } + return SDValue(); } bool -HexagonTargetLowering::isHvxOperation(SDValue Op) const { - // If the type of the result, or any operand type are HVX vector types, - // this is an HVX operation. - return Subtarget.isHVXVectorType(ty(Op), true) || - llvm::any_of(Op.getNode()->ops(), - [this] (SDValue V) { - return Subtarget.isHVXVectorType(ty(V), true); - }); +HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const { + auto Action = getPreferredHvxVectorAction(Ty); + if (Action == TargetLoweringBase::TypeWidenVector) { + EVT WideTy = getTypeToTransformTo(*DAG.getContext(), Ty); + assert(WideTy.isSimple()); + return Subtarget.isHVXVectorType(WideTy.getSimpleVT(), true); + } + return false; } bool -HexagonTargetLowering::isHvxOperation(SDNode *N) const { +HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const { + if (!Subtarget.useHVXOps()) + return false; // If the type of any result, or any operand type are HVX vector types, // this is an HVX operation. - auto IsHvxTy = [this] (EVT Ty) { + auto IsHvxTy = [this](EVT Ty) { return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true); }; - auto IsHvxOp = [this] (SDValue Op) { - return Subtarget.isHVXVectorType(ty(Op), true); + auto IsHvxOp = [this](SDValue Op) { + return Op.getValueType().isSimple() && + Subtarget.isHVXVectorType(ty(Op), true); + }; + if (llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp)) + return true; + + // Check if this could be an HVX operation after type widening. + auto IsWidenedToHvx = [this, &DAG](SDValue Op) { + if (!Op.getValueType().isSimple()) + return false; + MVT ValTy = ty(Op); + return ValTy.isVector() && shouldWidenToHvx(ValTy, DAG); }; - return llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp); + + for (int i = 0, e = N->getNumValues(); i != e; ++i) { + if (IsWidenedToHvx(SDValue(N, i))) + return true; + } + return llvm::any_of(N->ops(), IsWidenedToHvx); } diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp index d1cd23c3be3e..26fc093d15a7 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -1639,8 +1639,9 @@ bool HexagonInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1, return false; } -bool HexagonInstrInfo::DefinesPredicate(MachineInstr &MI, - std::vector<MachineOperand> &Pred) const { +bool HexagonInstrInfo::ClobbersPredicate(MachineInstr &MI, + std::vector<MachineOperand> &Pred, + bool SkipDead) const { const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); for (unsigned oper = 0; oper < MI.getNumOperands(); ++oper) { @@ -2721,6 +2722,8 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset, case Hexagon::PS_vloadrw_nt_ai: case Hexagon::V6_vL32b_ai: case Hexagon::V6_vS32b_ai: + case Hexagon::V6_vS32b_qpred_ai: + case Hexagon::V6_vS32b_nqpred_ai: case Hexagon::V6_vL32b_nt_ai: case Hexagon::V6_vS32b_nt_ai: case Hexagon::V6_vL32Ub_ai: diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h index 847b9a672891..11717996935d 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h @@ -238,8 +238,8 @@ public: /// If the specified instruction defines any predicate /// or condition code register(s) used for predication, returns true as well /// as the definition predicate(s) by reference. - bool DefinesPredicate(MachineInstr &MI, - std::vector<MachineOperand> &Pred) const override; + bool ClobbersPredicate(MachineInstr &MI, std::vector<MachineOperand> &Pred, + bool SkipDead) const override; /// Return true if the specified instruction can be predicated. /// By default, this returns true for every instruction with a diff --git a/llvm/lib/Target/Hexagon/HexagonIntrinsicsV60.td b/llvm/lib/Target/Hexagon/HexagonIntrinsicsV60.td index 1245ee7974b5..796979e59061 100644 --- a/llvm/lib/Target/Hexagon/HexagonIntrinsicsV60.td +++ b/llvm/lib/Target/Hexagon/HexagonIntrinsicsV60.td @@ -1,4 +1,4 @@ -//=- HexagonIntrinsicsV60.td - Target Description for Hexagon -*- tablegen *-=// +//===- HexagonIntrinsicsV60.td - V60 instruction intrinsics -*- tablegen *-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp index 2c1e0cadd9ee..76cc8f402c5a 100644 --- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp +++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "HexagonLoopIdiomRecognition.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" @@ -16,6 +17,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/MemoryLocation.h" @@ -40,6 +42,7 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsHexagon.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -108,136 +111,151 @@ static const char *HexagonVolatileMemcpyName namespace llvm { - void initializeHexagonLoopIdiomRecognizePass(PassRegistry&); - Pass *createHexagonLoopIdiomPass(); +void initializeHexagonLoopIdiomRecognizeLegacyPassPass(PassRegistry &); +Pass *createHexagonLoopIdiomPass(); } // end namespace llvm namespace { - class HexagonLoopIdiomRecognize : public LoopPass { - public: - static char ID; +class HexagonLoopIdiomRecognize { +public: + explicit HexagonLoopIdiomRecognize(AliasAnalysis *AA, DominatorTree *DT, + LoopInfo *LF, const TargetLibraryInfo *TLI, + ScalarEvolution *SE) + : AA(AA), DT(DT), LF(LF), TLI(TLI), SE(SE) {} - explicit HexagonLoopIdiomRecognize() : LoopPass(ID) { - initializeHexagonLoopIdiomRecognizePass(*PassRegistry::getPassRegistry()); - } + bool run(Loop *L); - StringRef getPassName() const override { - return "Recognize Hexagon-specific loop idioms"; - } +private: + int getSCEVStride(const SCEVAddRecExpr *StoreEv); + bool isLegalStore(Loop *CurLoop, StoreInst *SI); + void collectStores(Loop *CurLoop, BasicBlock *BB, + SmallVectorImpl<StoreInst *> &Stores); + bool processCopyingStore(Loop *CurLoop, StoreInst *SI, const SCEV *BECount); + bool coverLoop(Loop *L, SmallVectorImpl<Instruction *> &Insts) const; + bool runOnLoopBlock(Loop *CurLoop, BasicBlock *BB, const SCEV *BECount, + SmallVectorImpl<BasicBlock *> &ExitBlocks); + bool runOnCountableLoop(Loop *L); - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired<LoopInfoWrapperPass>(); - AU.addRequiredID(LoopSimplifyID); - AU.addRequiredID(LCSSAID); - AU.addRequired<AAResultsWrapperPass>(); - AU.addPreserved<AAResultsWrapperPass>(); - AU.addRequired<ScalarEvolutionWrapperPass>(); - AU.addRequired<DominatorTreeWrapperPass>(); - AU.addRequired<TargetLibraryInfoWrapperPass>(); - AU.addPreserved<TargetLibraryInfoWrapperPass>(); - } + AliasAnalysis *AA; + const DataLayout *DL; + DominatorTree *DT; + LoopInfo *LF; + const TargetLibraryInfo *TLI; + ScalarEvolution *SE; + bool HasMemcpy, HasMemmove; +}; - bool runOnLoop(Loop *L, LPPassManager &LPM) override; +class HexagonLoopIdiomRecognizeLegacyPass : public LoopPass { +public: + static char ID; - private: - int getSCEVStride(const SCEVAddRecExpr *StoreEv); - bool isLegalStore(Loop *CurLoop, StoreInst *SI); - void collectStores(Loop *CurLoop, BasicBlock *BB, - SmallVectorImpl<StoreInst*> &Stores); - bool processCopyingStore(Loop *CurLoop, StoreInst *SI, const SCEV *BECount); - bool coverLoop(Loop *L, SmallVectorImpl<Instruction*> &Insts) const; - bool runOnLoopBlock(Loop *CurLoop, BasicBlock *BB, const SCEV *BECount, - SmallVectorImpl<BasicBlock*> &ExitBlocks); - bool runOnCountableLoop(Loop *L); + explicit HexagonLoopIdiomRecognizeLegacyPass() : LoopPass(ID) { + initializeHexagonLoopIdiomRecognizeLegacyPassPass( + *PassRegistry::getPassRegistry()); + } - AliasAnalysis *AA; - const DataLayout *DL; - DominatorTree *DT; - LoopInfo *LF; - const TargetLibraryInfo *TLI; - ScalarEvolution *SE; - bool HasMemcpy, HasMemmove; - }; + StringRef getPassName() const override { + return "Recognize Hexagon-specific loop idioms"; + } - struct Simplifier { - struct Rule { - using FuncType = std::function<Value* (Instruction*, LLVMContext&)>; - Rule(StringRef N, FuncType F) : Name(N), Fn(F) {} - StringRef Name; // For debugging. - FuncType Fn; - }; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<LoopInfoWrapperPass>(); + AU.addRequiredID(LoopSimplifyID); + AU.addRequiredID(LCSSAID); + AU.addRequired<AAResultsWrapperPass>(); + AU.addRequired<ScalarEvolutionWrapperPass>(); + AU.addRequired<DominatorTreeWrapperPass>(); + AU.addRequired<TargetLibraryInfoWrapperPass>(); + AU.addPreserved<TargetLibraryInfoWrapperPass>(); + } - void addRule(StringRef N, const Rule::FuncType &F) { - Rules.push_back(Rule(N, F)); - } + bool runOnLoop(Loop *L, LPPassManager &LPM) override; +}; - private: - struct WorkListType { - WorkListType() = default; +struct Simplifier { + struct Rule { + using FuncType = std::function<Value *(Instruction *, LLVMContext &)>; + Rule(StringRef N, FuncType F) : Name(N), Fn(F) {} + StringRef Name; // For debugging. + FuncType Fn; + }; - void push_back(Value* V) { - // Do not push back duplicates. - if (!S.count(V)) { Q.push_back(V); S.insert(V); } - } + void addRule(StringRef N, const Rule::FuncType &F) { + Rules.push_back(Rule(N, F)); + } + +private: + struct WorkListType { + WorkListType() = default; - Value *pop_front_val() { - Value *V = Q.front(); Q.pop_front(); S.erase(V); - return V; + void push_back(Value *V) { + // Do not push back duplicates. + if (!S.count(V)) { + Q.push_back(V); + S.insert(V); } + } - bool empty() const { return Q.empty(); } + Value *pop_front_val() { + Value *V = Q.front(); + Q.pop_front(); + S.erase(V); + return V; + } - private: - std::deque<Value*> Q; - std::set<Value*> S; - }; + bool empty() const { return Q.empty(); } + + private: + std::deque<Value *> Q; + std::set<Value *> S; + }; - using ValueSetType = std::set<Value *>; + using ValueSetType = std::set<Value *>; - std::vector<Rule> Rules; + std::vector<Rule> Rules; - public: - struct Context { - using ValueMapType = DenseMap<Value *, Value *>; +public: + struct Context { + using ValueMapType = DenseMap<Value *, Value *>; - Value *Root; - ValueSetType Used; // The set of all cloned values used by Root. - ValueSetType Clones; // The set of all cloned values. - LLVMContext &Ctx; + Value *Root; + ValueSetType Used; // The set of all cloned values used by Root. + ValueSetType Clones; // The set of all cloned values. + LLVMContext &Ctx; - Context(Instruction *Exp) + Context(Instruction *Exp) : Ctx(Exp->getParent()->getParent()->getContext()) { - initialize(Exp); - } - - ~Context() { cleanup(); } + initialize(Exp); + } - void print(raw_ostream &OS, const Value *V) const; - Value *materialize(BasicBlock *B, BasicBlock::iterator At); + ~Context() { cleanup(); } - private: - friend struct Simplifier; + void print(raw_ostream &OS, const Value *V) const; + Value *materialize(BasicBlock *B, BasicBlock::iterator At); - void initialize(Instruction *Exp); - void cleanup(); + private: + friend struct Simplifier; - template <typename FuncT> void traverse(Value *V, FuncT F); - void record(Value *V); - void use(Value *V); - void unuse(Value *V); + void initialize(Instruction *Exp); + void cleanup(); - bool equal(const Instruction *I, const Instruction *J) const; - Value *find(Value *Tree, Value *Sub) const; - Value *subst(Value *Tree, Value *OldV, Value *NewV); - void replace(Value *OldV, Value *NewV); - void link(Instruction *I, BasicBlock *B, BasicBlock::iterator At); - }; + template <typename FuncT> void traverse(Value *V, FuncT F); + void record(Value *V); + void use(Value *V); + void unuse(Value *V); - Value *simplify(Context &C); + bool equal(const Instruction *I, const Instruction *J) const; + Value *find(Value *Tree, Value *Sub) const; + Value *subst(Value *Tree, Value *OldV, Value *NewV); + void replace(Value *OldV, Value *NewV); + void link(Instruction *I, BasicBlock *B, BasicBlock::iterator At); }; + Value *simplify(Context &C); +}; + struct PE { PE(const Simplifier::Context &c, Value *v = nullptr) : C(c), V(v) {} @@ -253,10 +271,10 @@ namespace { } // end anonymous namespace -char HexagonLoopIdiomRecognize::ID = 0; +char HexagonLoopIdiomRecognizeLegacyPass::ID = 0; -INITIALIZE_PASS_BEGIN(HexagonLoopIdiomRecognize, "hexagon-loop-idiom", - "Recognize Hexagon-specific loop idioms", false, false) +INITIALIZE_PASS_BEGIN(HexagonLoopIdiomRecognizeLegacyPass, "hexagon-loop-idiom", + "Recognize Hexagon-specific loop idioms", false, false) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(LoopSimplify) INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass) @@ -264,8 +282,8 @@ INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) -INITIALIZE_PASS_END(HexagonLoopIdiomRecognize, "hexagon-loop-idiom", - "Recognize Hexagon-specific loop idioms", false, false) +INITIALIZE_PASS_END(HexagonLoopIdiomRecognizeLegacyPass, "hexagon-loop-idiom", + "Recognize Hexagon-specific loop idioms", false, false) template <typename FuncT> void Simplifier::Context::traverse(Value *V, FuncT F) { @@ -1973,7 +1991,7 @@ mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L, // Get the location that may be stored across the loop. Since the access // is strided positively through memory, we say that the modified location // starts at the pointer and has infinite size. - LocationSize AccessSize = LocationSize::unknown(); + LocationSize AccessSize = LocationSize::afterPointer(); // If the loop iterates a fixed number of times, we can refine the access // size to be exactly the size of the memset, which is (BECount+1)*StoreSize @@ -2404,14 +2422,11 @@ bool HexagonLoopIdiomRecognize::runOnCountableLoop(Loop *L) { return Changed; } -bool HexagonLoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) { +bool HexagonLoopIdiomRecognize::run(Loop *L) { const Module &M = *L->getHeader()->getParent()->getParent(); if (Triple(M.getTargetTriple()).getArch() != Triple::hexagon) return false; - if (skipLoop(L)) - return false; - // If the loop could not be converted to canonical form, it must have an // indirectbr in it, just give up. if (!L->getLoopPreheader()) @@ -2422,13 +2437,7 @@ bool HexagonLoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) { if (Name == "memset" || Name == "memcpy" || Name == "memmove") return false; - AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); DL = &L->getHeader()->getModule()->getDataLayout(); - DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); - LF = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); - TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI( - *L->getHeader()->getParent()); - SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); HasMemcpy = TLI->has(LibFunc_memcpy); HasMemmove = TLI->has(LibFunc_memmove); @@ -2438,6 +2447,30 @@ bool HexagonLoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) { return false; } +bool HexagonLoopIdiomRecognizeLegacyPass::runOnLoop(Loop *L, + LPPassManager &LPM) { + if (skipLoop(L)) + return false; + + auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); + auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); + auto *LF = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); + auto *TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI( + *L->getHeader()->getParent()); + auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); + return HexagonLoopIdiomRecognize(AA, DT, LF, TLI, SE).run(L); +} + Pass *llvm::createHexagonLoopIdiomPass() { - return new HexagonLoopIdiomRecognize(); + return new HexagonLoopIdiomRecognizeLegacyPass(); +} + +PreservedAnalyses +HexagonLoopIdiomRecognitionPass::run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U) { + return HexagonLoopIdiomRecognize(&AR.AA, &AR.DT, &AR.LI, &AR.TLI, &AR.SE) + .run(&L) + ? getLoopPassPreservedAnalyses() + : PreservedAnalyses::all(); } diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.h b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.h new file mode 100644 index 000000000000..28ec83b05dac --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.h @@ -0,0 +1,24 @@ +//===- HexagonLoopIdiomRecognition.h --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONLOOPIDIOMRECOGNITION_H +#define LLVM_LIB_TARGET_HEXAGON_HEXAGONLOOPIDIOMRECOGNITION_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" + +namespace llvm { + +struct HexagonLoopIdiomRecognitionPass + : PassInfoMixin<HexagonLoopIdiomRecognitionPass> { + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); +}; +} // namespace llvm + +#endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONLOOPIDIOMRECOGNITION_H diff --git a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp index 188d91355a35..9507de95231f 100644 --- a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp +++ b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp @@ -104,7 +104,7 @@ void llvm::HexagonLowerToMC(const MCInstrInfo &MCII, const MachineInstr *MI, HexagonMCInstrInfo::setOuterLoop(MCB); return; } - MCInst *MCI = new (AP.OutContext) MCInst; + MCInst *MCI = AP.OutContext.createMCInst(); MCI->setOpcode(MI->getOpcode()); assert(MCI->getOpcode() == static_cast<unsigned>(MI->getOpcode()) && "MCI opcode should have been set on construction"); diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp index c718e5f2d9fb..2cdfbe7845b6 100644 --- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp +++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp @@ -246,7 +246,7 @@ void HexagonOptAddrMode::getAllRealUses(NodeAddr<StmtNode *> SA, for (NodeAddr<DefNode *> DA : SA.Addr->members_if(DFG->IsDef, *DFG)) { LLVM_DEBUG(dbgs() << "\t\t[DefNode]: " << Print<NodeAddr<DefNode *>>(DA, *DFG) << "\n"); - RegisterRef DR = DFG->getPRI().normalize(DA.Addr->getRegRef(*DFG)); + RegisterRef DR = DA.Addr->getRegRef(*DFG); auto UseSet = LV->getAllReachedUses(DR, DA); diff --git a/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp b/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp index d818e0897f75..e026bb6d601d 100644 --- a/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp +++ b/llvm/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp @@ -11,7 +11,9 @@ // //===----------------------------------------------------------------------===// +#include "Hexagon.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -19,8 +21,6 @@ #include "llvm/Pass.h" #include "llvm/Transforms/Scalar.h" -#include "Hexagon.h" - using namespace llvm; namespace llvm { diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td index cc10627955fb..d216c511a994 100644 --- a/llvm/lib/Target/Hexagon/HexagonPatterns.td +++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -1,4 +1,4 @@ -//==- HexagonPatterns.td - Target Description for Hexagon -*- tablegen -*-===// +//===- HexagonPatterns.td - Selection Patterns for Hexagon -*- tablegen -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -229,6 +229,21 @@ def NegImm32: SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(NV, SDLoc(N), MVT::i32); }]>; +def SplatB: SDNodeXForm<imm, [{ + uint32_t V = N->getZExtValue(); + assert(isUInt<8>(V) || V >> 8 == 0xFFFFFF); + V &= 0xFF; + uint32_t S = V << 24 | V << 16 | V << 8 | V; + return CurDAG->getTargetConstant(S, SDLoc(N), MVT::i32); +}]>; + +def SplatH: SDNodeXForm<imm, [{ + uint32_t V = N->getZExtValue(); + assert(isUInt<16>(V) || V >> 16 == 0xFFFF); + V &= 0xFFFF; + return CurDAG->getTargetConstant(V << 16 | V, SDLoc(N), MVT::i32); +}]>; + // Helpers for type promotions/contractions. def I1toI32: OutPatFrag<(ops node:$Rs), (C2_muxii (i1 $Rs), 1, 0)>; @@ -351,12 +366,14 @@ multiclass NopCast_pat<ValueType Ty1, ValueType Ty2, RegisterClass RC> { def: Pat<(Ty2 (bitconvert (Ty1 RC:$Val))), (Ty2 RC:$Val)>; } - // Frags for commonly used SDNodes. def Add: pf2<add>; def And: pf2<and>; def Sra: pf2<sra>; def Sub: pf2<sub>; def Or: pf2<or>; def Srl: pf2<srl>; def Mul: pf2<mul>; def Xor: pf2<xor>; def Shl: pf2<shl>; +def Smin: pf2<smin>; def Smax: pf2<smax>; +def Umin: pf2<umin>; def Umax: pf2<umax>; + def Rol: pf2<rotl>; // --(1) Immediate ------------------------------------------------------- @@ -909,25 +926,14 @@ let AddedComplexity = 200 in { defm: SelMinMax16_pats<setult, A2_minu, A2_maxu>; } -let AddedComplexity = 200 in { - defm: MinMax_pats<A2_min, A2_max, select, setgt, i1, I32>; - defm: MinMax_pats<A2_min, A2_max, select, setge, i1, I32>; - defm: MinMax_pats<A2_max, A2_min, select, setlt, i1, I32>; - defm: MinMax_pats<A2_max, A2_min, select, setle, i1, I32>; - defm: MinMax_pats<A2_minu, A2_maxu, select, setugt, i1, I32>; - defm: MinMax_pats<A2_minu, A2_maxu, select, setuge, i1, I32>; - defm: MinMax_pats<A2_maxu, A2_minu, select, setult, i1, I32>; - defm: MinMax_pats<A2_maxu, A2_minu, select, setule, i1, I32>; - - defm: MinMax_pats<A2_minp, A2_maxp, select, setgt, i1, I64>; - defm: MinMax_pats<A2_minp, A2_maxp, select, setge, i1, I64>; - defm: MinMax_pats<A2_maxp, A2_minp, select, setlt, i1, I64>; - defm: MinMax_pats<A2_maxp, A2_minp, select, setle, i1, I64>; - defm: MinMax_pats<A2_minup, A2_maxup, select, setugt, i1, I64>; - defm: MinMax_pats<A2_minup, A2_maxup, select, setuge, i1, I64>; - defm: MinMax_pats<A2_maxup, A2_minup, select, setult, i1, I64>; - defm: MinMax_pats<A2_maxup, A2_minup, select, setule, i1, I64>; -} +def: OpR_RR_pat<A2_min, Smin, i32, I32, I32>; +def: OpR_RR_pat<A2_max, Smax, i32, I32, I32>; +def: OpR_RR_pat<A2_minu, Umin, i32, I32, I32>; +def: OpR_RR_pat<A2_maxu, Umax, i32, I32, I32>; +def: OpR_RR_pat<A2_minp, Smin, i64, I64, I64>; +def: OpR_RR_pat<A2_maxp, Smax, i64, I64, I64>; +def: OpR_RR_pat<A2_minup, Umin, i64, I64, I64>; +def: OpR_RR_pat<A2_maxup, Umax, i64, I64, I64>; let AddedComplexity = 100 in { defm: MinMax_pats<F2_sfmin, F2_sfmax, select, setogt, i1, F32>; @@ -943,18 +949,20 @@ let AddedComplexity = 100, Predicates = [HasV67] in { defm: MinMax_pats<F2_dfmax, F2_dfmin, select, setole, i1, F64>; } -defm: MinMax_pats<A2_vminb, A2_vmaxb, vselect, setgt, v8i1, V8I8>; -defm: MinMax_pats<A2_vminb, A2_vmaxb, vselect, setge, v8i1, V8I8>; -defm: MinMax_pats<A2_vminh, A2_vmaxh, vselect, setgt, v4i1, V4I16>; -defm: MinMax_pats<A2_vminh, A2_vmaxh, vselect, setge, v4i1, V4I16>; -defm: MinMax_pats<A2_vminw, A2_vmaxw, vselect, setgt, v2i1, V2I32>; -defm: MinMax_pats<A2_vminw, A2_vmaxw, vselect, setge, v2i1, V2I32>; -defm: MinMax_pats<A2_vminub, A2_vmaxub, vselect, setugt, v8i1, V8I8>; -defm: MinMax_pats<A2_vminub, A2_vmaxub, vselect, setuge, v8i1, V8I8>; -defm: MinMax_pats<A2_vminuh, A2_vmaxuh, vselect, setugt, v4i1, V4I16>; -defm: MinMax_pats<A2_vminuh, A2_vmaxuh, vselect, setuge, v4i1, V4I16>; -defm: MinMax_pats<A2_vminuw, A2_vmaxuw, vselect, setugt, v2i1, V2I32>; -defm: MinMax_pats<A2_vminuw, A2_vmaxuw, vselect, setuge, v2i1, V2I32>; +def: OpR_RR_pat<A2_vminb, Smin, v8i8, V8I8>; +def: OpR_RR_pat<A2_vmaxb, Smax, v8i8, V8I8>; +def: OpR_RR_pat<A2_vminub, Umin, v8i8, V8I8>; +def: OpR_RR_pat<A2_vmaxub, Umax, v8i8, V8I8>; + +def: OpR_RR_pat<A2_vminh, Smin, v4i16, V4I16>; +def: OpR_RR_pat<A2_vmaxh, Smax, v4i16, V4I16>; +def: OpR_RR_pat<A2_vminuh, Umin, v4i16, V4I16>; +def: OpR_RR_pat<A2_vmaxuh, Umax, v4i16, V4I16>; + +def: OpR_RR_pat<A2_vminw, Smin, v2i32, V2I32>; +def: OpR_RR_pat<A2_vmaxw, Smax, v2i32, V2I32>; +def: OpR_RR_pat<A2_vminuw, Umin, v2i32, V2I32>; +def: OpR_RR_pat<A2_vmaxuw, Umax, v2i32, V2I32>; // --(7) Insert/extract -------------------------------------------------- // @@ -991,21 +999,26 @@ def: Pat<(HexagonEXTRACTU I32:$Rs, I32:$Width, I32:$Off), def: Pat<(HexagonEXTRACTU I64:$Rs, I32:$Width, I32:$Off), (S2_extractup_rp I64:$Rs, (Combinew $Width, $Off))>; -def SDTHexagonVSPLAT: - SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; - -def HexagonVSPLAT: SDNode<"HexagonISD::VSPLAT", SDTHexagonVSPLAT>; - -def: Pat<(v4i8 (HexagonVSPLAT I32:$Rs)), (S2_vsplatrb I32:$Rs)>; -def: Pat<(v4i16 (HexagonVSPLAT I32:$Rs)), (S2_vsplatrh I32:$Rs)>; -def: Pat<(v2i32 (HexagonVSPLAT s8_0ImmPred:$s8)), +def: Pat<(v4i8 (splat_vector anyint:$V)), (ToI32 (SplatB $V))>; +def: Pat<(v2i16 (splat_vector anyint:$V)), (ToI32 (SplatH $V))>; +def: Pat<(v8i8 (splat_vector anyint:$V)), + (Combinew (ToI32 (SplatB $V)), (ToI32 (SplatB $V)))>; +def: Pat<(v4i16 (splat_vector anyint:$V)), + (Combinew (ToI32 (SplatH $V)), (ToI32 (SplatH $V)))>; +let AddedComplexity = 10 in +def: Pat<(v2i32 (splat_vector s8_0ImmPred:$s8)), (A2_combineii imm:$s8, imm:$s8)>; -def: Pat<(v2i32 (HexagonVSPLAT I32:$Rs)), (Combinew I32:$Rs, I32:$Rs)>; +def: Pat<(v2i32 (splat_vector anyimm:$V)), (Combinew (ToI32 $V), (ToI32 $V))>; + +def: Pat<(v4i8 (splat_vector I32:$Rs)), (S2_vsplatrb I32:$Rs)>; +def: Pat<(v2i16 (splat_vector I32:$Rs)), (LoReg (S2_vsplatrh I32:$Rs))>; +def: Pat<(v4i16 (splat_vector I32:$Rs)), (S2_vsplatrh I32:$Rs)>; +def: Pat<(v2i32 (splat_vector I32:$Rs)), (Combinew I32:$Rs, I32:$Rs)>; let AddedComplexity = 10 in -def: Pat<(v8i8 (HexagonVSPLAT I32:$Rs)), (S6_vsplatrbp I32:$Rs)>, +def: Pat<(v8i8 (splat_vector I32:$Rs)), (S6_vsplatrbp I32:$Rs)>, Requires<[HasV62]>; -def: Pat<(v8i8 (HexagonVSPLAT I32:$Rs)), +def: Pat<(v8i8 (splat_vector I32:$Rs)), (Combinew (S2_vsplatrb I32:$Rs), (S2_vsplatrb I32:$Rs))>; @@ -1082,9 +1095,9 @@ def FShl32r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru), (HiReg (S2_asl_r_p (Combinew $Rs, $Rt), $Ru))>; def FShl64i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S), - (S2_lsr_i_p_or (S2_asl_i_p $Rt, $S), $Rs, (Subi<64> $S))>; + (S2_lsr_i_p_or (S2_asl_i_p $Rs, $S), $Rt, (Subi<64> $S))>; def FShl64r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru), - (S2_lsr_r_p_or (S2_asl_r_p $Rt, $Ru), $Rs, (A2_subri 64, $Ru))>; + (S2_lsr_r_p_or (S2_asl_r_p $Rs, $Ru), $Rt, (A2_subri 64, $Ru))>; // Combined SDNodeXForm: (Divu8 (Subi<64> $S)) def Divu64_8: SDNodeXForm<imm, [{ @@ -1307,17 +1320,17 @@ def: OpR_RR_pat<S2_asr_r_vh, pf2<HexagonVASR>, v4i16, V4I16, I32>; def: OpR_RR_pat<S2_lsr_r_vw, pf2<HexagonVLSR>, v2i32, V2I32, I32>; def: OpR_RR_pat<S2_lsr_r_vh, pf2<HexagonVLSR>, v4i16, V4I16, I32>; -def: Pat<(sra V2I32:$b, (v2i32 (HexagonVSPLAT u5_0ImmPred:$c))), +def: Pat<(sra V2I32:$b, (v2i32 (splat_vector u5_0ImmPred:$c))), (S2_asr_i_vw V2I32:$b, imm:$c)>; -def: Pat<(srl V2I32:$b, (v2i32 (HexagonVSPLAT u5_0ImmPred:$c))), +def: Pat<(srl V2I32:$b, (v2i32 (splat_vector u5_0ImmPred:$c))), (S2_lsr_i_vw V2I32:$b, imm:$c)>; -def: Pat<(shl V2I32:$b, (v2i32 (HexagonVSPLAT u5_0ImmPred:$c))), +def: Pat<(shl V2I32:$b, (v2i32 (splat_vector u5_0ImmPred:$c))), (S2_asl_i_vw V2I32:$b, imm:$c)>; -def: Pat<(sra V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c))), +def: Pat<(sra V4I16:$b, (v4i16 (splat_vector u4_0ImmPred:$c))), (S2_asr_i_vh V4I16:$b, imm:$c)>; -def: Pat<(srl V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c))), +def: Pat<(srl V4I16:$b, (v4i16 (splat_vector u4_0ImmPred:$c))), (S2_lsr_i_vh V4I16:$b, imm:$c)>; -def: Pat<(shl V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c))), +def: Pat<(shl V4I16:$b, (v4i16 (splat_vector u4_0ImmPred:$c))), (S2_asl_i_vh V4I16:$b, imm:$c)>; def: Pat<(HexagonVASR V2I16:$Rs, u4_0ImmPred:$S), @@ -1688,8 +1701,6 @@ def: Pat<(fma F32:$Rs, F32:$Rt, F32:$Rx), (F2_sffma F32:$Rx, F32:$Rs, F32:$Rt)>; def: Pat<(fma (fneg F32:$Rs), F32:$Rt, F32:$Rx), (F2_sffms F32:$Rx, F32:$Rs, F32:$Rt)>; -def: Pat<(fma F32:$Rs, (fneg F32:$Rt), F32:$Rx), - (F2_sffms F32:$Rx, F32:$Rs, F32:$Rt)>; def: Pat<(mul V2I32:$Rs, V2I32:$Rt), (PS_vmulw V2I32:$Rs, V2I32:$Rt)>; diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td index 078a7135c55b..cd894c555adc 100644 --- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td +++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td @@ -1,3 +1,15 @@ +//===- HexagonPatternsHVX.td - Selection Patterns for HVX --*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +def SDTVecUnaryOp: + SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>; + def SDTVecBinOp: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<1,2>]>; @@ -9,9 +21,6 @@ def SDTHexagonVINSERTW0: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]>; def HexagonVINSERTW0: SDNode<"HexagonISD::VINSERTW0", SDTHexagonVINSERTW0>; -def SDTHexagonVSPLATW: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; -def HexagonVSPLATW: SDNode<"HexagonISD::VSPLATW", SDTHexagonVSPLATW>; - def HwLen2: SDNodeXForm<imm, [{ const auto &ST = static_cast<const HexagonSubtarget&>(CurDAG->getSubtarget()); return CurDAG->getTargetConstant(ST.getVectorLength()/2, SDLoc(N), MVT::i32); @@ -33,37 +42,29 @@ def Combineq: OutPatFrag<(ops node:$Qs, node:$Qt), def LoVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_lo)>; def HiVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_hi)>; -def HexagonVZERO: SDNode<"HexagonISD::VZERO", SDTVecLeaf>; def HexagonQCAT: SDNode<"HexagonISD::QCAT", SDTVecBinOp>; def HexagonQTRUE: SDNode<"HexagonISD::QTRUE", SDTVecLeaf>; def HexagonQFALSE: SDNode<"HexagonISD::QFALSE", SDTVecLeaf>; +def HexagonVPACKL: SDNode<"HexagonISD::VPACKL", SDTVecUnaryOp>; +def HexagonVUNPACK: SDNode<"HexagonISD::VUNPACK", SDTVecUnaryOp>; +def HexagonVUNPACKU: SDNode<"HexagonISD::VUNPACKU", SDTVecUnaryOp>; -def vzero: PatFrag<(ops), (HexagonVZERO)>; +def vzero: PatFrag<(ops), (splat_vector (i32 0))>; def qtrue: PatFrag<(ops), (HexagonQTRUE)>; def qfalse: PatFrag<(ops), (HexagonQFALSE)>; def qcat: PatFrag<(ops node:$Qs, node:$Qt), (HexagonQCAT node:$Qs, node:$Qt)>; -def qnot: PatFrag<(ops node:$Qs), (xor node:$Qs, qtrue)>; +def qnot: PatFrag<(ops node:$Qs), (xor node:$Qs, qtrue)>; +def vpackl: PatFrag<(ops node:$Vs), (HexagonVPACKL node:$Vs)>; +def vunpack: PatFrag<(ops node:$Vs), (HexagonVUNPACK node:$Vs)>; +def vunpacku: PatFrag<(ops node:$Vs), (HexagonVUNPACKU node:$Vs)>; def VSxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackb $Vs)>; def VSxth: OutPatFrag<(ops node:$Vs), (V6_vunpackh $Vs)>; def VZxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackub $Vs)>; def VZxth: OutPatFrag<(ops node:$Vs), (V6_vunpackuh $Vs)>; -def SplatB: SDNodeXForm<imm, [{ - uint32_t V = N->getZExtValue(); - assert(isUInt<8>(V)); - uint32_t S = V << 24 | V << 16 | V << 8 | V; - return CurDAG->getTargetConstant(S, SDLoc(N), MVT::i32); -}]>; - -def SplatH: SDNodeXForm<imm, [{ - uint32_t V = N->getZExtValue(); - assert(isUInt<16>(V)); - return CurDAG->getTargetConstant(V << 16 | V, SDLoc(N), MVT::i32); -}]>; - def IsVecOff : PatLeaf<(i32 imm), [{ int32_t V = N->getSExtValue(); int32_t VecSize = HRI->getSpillSize(Hexagon::HvxVRRegClass); @@ -171,16 +172,19 @@ let Predicates = [UseHVX] in { } let Predicates = [UseHVX] in { - def: Pat<(VecI8 vzero), (V6_vd0)>; - def: Pat<(VecI16 vzero), (V6_vd0)>; - def: Pat<(VecI32 vzero), (V6_vd0)>; - def: Pat<(VecPI8 vzero), (PS_vdd0)>; - def: Pat<(VecPI16 vzero), (PS_vdd0)>; - def: Pat<(VecPI32 vzero), (PS_vdd0)>; + let AddedComplexity = 100 in { + // These should be preferred over a vsplat of 0. + def: Pat<(VecI8 vzero), (V6_vd0)>; + def: Pat<(VecI16 vzero), (V6_vd0)>; + def: Pat<(VecI32 vzero), (V6_vd0)>; + def: Pat<(VecPI8 vzero), (PS_vdd0)>; + def: Pat<(VecPI16 vzero), (PS_vdd0)>; + def: Pat<(VecPI32 vzero), (PS_vdd0)>; - def: Pat<(concat_vectors (VecI8 vzero), (VecI8 vzero)), (PS_vdd0)>; - def: Pat<(concat_vectors (VecI16 vzero), (VecI16 vzero)), (PS_vdd0)>; - def: Pat<(concat_vectors (VecI32 vzero), (VecI32 vzero)), (PS_vdd0)>; + def: Pat<(concat_vectors (VecI8 vzero), (VecI8 vzero)), (PS_vdd0)>; + def: Pat<(concat_vectors (VecI16 vzero), (VecI16 vzero)), (PS_vdd0)>; + def: Pat<(concat_vectors (VecI32 vzero), (VecI32 vzero)), (PS_vdd0)>; + } def: Pat<(VecPI8 (concat_vectors HVI8:$Vs, HVI8:$Vt)), (Combinev HvxVR:$Vt, HvxVR:$Vs)>; @@ -207,63 +211,70 @@ let Predicates = [UseHVX] in { (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; } -def Vsplatib: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 (SplatB $V)))>; -def Vsplatih: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 (SplatH $V)))>; -def Vsplatiw: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 $V))>; +// Splats for HvxV60 +def V60splatib: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 (SplatB $V)))>; +def V60splatih: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 (SplatH $V)))>; +def V60splatiw: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 $V))>; +def V60splatrb: OutPatFrag<(ops node:$Rs), (V6_lvsplatw (S2_vsplatrb $Rs))>; +def V60splatrh: OutPatFrag<(ops node:$Rs), + (V6_lvsplatw (A2_combine_ll $Rs, $Rs))>; +def V60splatrw: OutPatFrag<(ops node:$Rs), (V6_lvsplatw $Rs)>; -def Vsplatrb: OutPatFrag<(ops node:$Rs), (V6_lvsplatw (S2_vsplatrb $Rs))>; -def Vsplatrh: OutPatFrag<(ops node:$Rs), - (V6_lvsplatw (A2_combine_ll $Rs, $Rs))>; -def Vsplatrw: OutPatFrag<(ops node:$Rs), (V6_lvsplatw $Rs)>; +// Splats for HvxV62+ +def V62splatib: OutPatFrag<(ops node:$V), (V6_lvsplatb (ToI32 $V))>; +def V62splatih: OutPatFrag<(ops node:$V), (V6_lvsplath (ToI32 $V))>; +def V62splatiw: OutPatFrag<(ops node:$V), (V6_lvsplatw (ToI32 $V))>; +def V62splatrb: OutPatFrag<(ops node:$Rs), (V6_lvsplatb $Rs)>; +def V62splatrh: OutPatFrag<(ops node:$Rs), (V6_lvsplath $Rs)>; +def V62splatrw: OutPatFrag<(ops node:$Rs), (V6_lvsplatw $Rs)>; def Rep: OutPatFrag<(ops node:$N), (Combinev $N, $N)>; -let Predicates = [UseHVX] in { +let Predicates = [UseHVX,UseHVXV60] in { let AddedComplexity = 10 in { - def: Pat<(VecI8 (HexagonVSPLAT u8_0ImmPred:$V)), (Vsplatib $V)>; - def: Pat<(VecI16 (HexagonVSPLAT u16_0ImmPred:$V)), (Vsplatih $V)>; - def: Pat<(VecI32 (HexagonVSPLAT anyimm:$V)), (Vsplatiw $V)>; - def: Pat<(VecPI8 (HexagonVSPLAT u8_0ImmPred:$V)), (Rep (Vsplatib $V))>; - def: Pat<(VecPI16 (HexagonVSPLAT u16_0ImmPred:$V)), (Rep (Vsplatih $V))>; - def: Pat<(VecPI32 (HexagonVSPLAT anyimm:$V)), (Rep (Vsplatiw $V))>; + def: Pat<(VecI8 (splat_vector u8_0ImmPred:$V)), (V60splatib $V)>; + def: Pat<(VecI16 (splat_vector u16_0ImmPred:$V)), (V60splatih $V)>; + def: Pat<(VecI32 (splat_vector anyimm:$V)), (V60splatiw $V)>; + def: Pat<(VecPI8 (splat_vector u8_0ImmPred:$V)), (Rep (V60splatib $V))>; + def: Pat<(VecPI16 (splat_vector u16_0ImmPred:$V)), (Rep (V60splatih $V))>; + def: Pat<(VecPI32 (splat_vector anyimm:$V)), (Rep (V60splatiw $V))>; + } + def: Pat<(VecI8 (splat_vector I32:$Rs)), (V60splatrb $Rs)>; + def: Pat<(VecI16 (splat_vector I32:$Rs)), (V60splatrh $Rs)>; + def: Pat<(VecI32 (splat_vector I32:$Rs)), (V60splatrw $Rs)>; + def: Pat<(VecPI8 (splat_vector I32:$Rs)), (Rep (V60splatrb $Rs))>; + def: Pat<(VecPI16 (splat_vector I32:$Rs)), (Rep (V60splatrh $Rs))>; + def: Pat<(VecPI32 (splat_vector I32:$Rs)), (Rep (V60splatrw $Rs))>; +} +let Predicates = [UseHVX,UseHVXV62] in { + let AddedComplexity = 30 in { + def: Pat<(VecI8 (splat_vector u8_0ImmPred:$V)), (V62splatib imm:$V)>; + def: Pat<(VecI16 (splat_vector u16_0ImmPred:$V)), (V62splatih imm:$V)>; + def: Pat<(VecI32 (splat_vector anyimm:$V)), (V62splatiw imm:$V)>; + def: Pat<(VecPI8 (splat_vector u8_0ImmPred:$V)), + (Rep (V62splatib imm:$V))>; + def: Pat<(VecPI16 (splat_vector u16_0ImmPred:$V)), + (Rep (V62splatih imm:$V))>; + def: Pat<(VecPI32 (splat_vector anyimm:$V)), + (Rep (V62splatiw imm:$V))>; + } + let AddedComplexity = 20 in { + def: Pat<(VecI8 (splat_vector I32:$Rs)), (V62splatrb $Rs)>; + def: Pat<(VecI16 (splat_vector I32:$Rs)), (V62splatrh $Rs)>; + def: Pat<(VecI32 (splat_vector I32:$Rs)), (V62splatrw $Rs)>; + def: Pat<(VecPI8 (splat_vector I32:$Rs)), (Rep (V62splatrb $Rs))>; + def: Pat<(VecPI16 (splat_vector I32:$Rs)), (Rep (V62splatrh $Rs))>; + def: Pat<(VecPI32 (splat_vector I32:$Rs)), (Rep (V62splatrw $Rs))>; } - def: Pat<(VecI8 (HexagonVSPLAT I32:$Rs)), (Vsplatrb $Rs)>; - def: Pat<(VecI16 (HexagonVSPLAT I32:$Rs)), (Vsplatrh $Rs)>; - def: Pat<(VecI32 (HexagonVSPLAT I32:$Rs)), (Vsplatrw $Rs)>; - def: Pat<(VecPI8 (HexagonVSPLAT I32:$Rs)), (Rep (Vsplatrb $Rs))>; - def: Pat<(VecPI16 (HexagonVSPLAT I32:$Rs)), (Rep (Vsplatrh $Rs))>; - def: Pat<(VecPI32 (HexagonVSPLAT I32:$Rs)), (Rep (Vsplatrw $Rs))>; - - def: Pat<(VecI8 (HexagonVSPLATW I32:$Rs)), (Vsplatrw $Rs)>; - def: Pat<(VecI16 (HexagonVSPLATW I32:$Rs)), (Vsplatrw $Rs)>; - def: Pat<(VecI32 (HexagonVSPLATW I32:$Rs)), (Vsplatrw $Rs)>; - def: Pat<(VecPI8 (HexagonVSPLATW I32:$Rs)), (Rep (Vsplatrw $Rs))>; - def: Pat<(VecPI16 (HexagonVSPLATW I32:$Rs)), (Rep (Vsplatrw $Rs))>; - def: Pat<(VecPI32 (HexagonVSPLATW I32:$Rs)), (Rep (Vsplatrw $Rs))>; } class Vneg1<ValueType VecTy> - : PatFrag<(ops), (VecTy (HexagonVSPLATW (i32 -1)))>; + : PatFrag<(ops), (VecTy (splat_vector (i32 -1)))>; class Vnot<ValueType VecTy> : PatFrag<(ops node:$Vs), (xor $Vs, Vneg1<VecTy>)>; let Predicates = [UseHVX] in { - let AddedComplexity = 220 in { - defm: MinMax_pats<V6_vminb, V6_vmaxb, vselect, setgt, VecQ8, HVI8>; - defm: MinMax_pats<V6_vminb, V6_vmaxb, vselect, setge, VecQ8, HVI8>; - defm: MinMax_pats<V6_vminub, V6_vmaxub, vselect, setugt, VecQ8, HVI8>; - defm: MinMax_pats<V6_vminub, V6_vmaxub, vselect, setuge, VecQ8, HVI8>; - defm: MinMax_pats<V6_vminh, V6_vmaxh, vselect, setgt, VecQ16, HVI16>; - defm: MinMax_pats<V6_vminh, V6_vmaxh, vselect, setge, VecQ16, HVI16>; - defm: MinMax_pats<V6_vminuh, V6_vmaxuh, vselect, setugt, VecQ16, HVI16>; - defm: MinMax_pats<V6_vminuh, V6_vmaxuh, vselect, setuge, VecQ16, HVI16>; - defm: MinMax_pats<V6_vminw, V6_vmaxw, vselect, setgt, VecQ32, HVI32>; - defm: MinMax_pats<V6_vminw, V6_vmaxw, vselect, setge, VecQ32, HVI32>; - } -} - -let Predicates = [UseHVX] in { let AddedComplexity = 200 in { def: Pat<(Vnot<VecI8> HVI8:$Vs), (V6_vnot HvxVR:$Vs)>; def: Pat<(Vnot<VecI16> HVI16:$Vs), (V6_vnot HvxVR:$Vs)>; @@ -292,6 +303,17 @@ let Predicates = [UseHVX] in { def: OpR_RR_pat<V6_vxor, Xor, VecI16, HVI16>; def: OpR_RR_pat<V6_vxor, Xor, VecI32, HVI32>; + def: OpR_RR_pat<V6_vminb, Smin, VecI8, HVI8>; + def: OpR_RR_pat<V6_vmaxb, Smax, VecI8, HVI8>; + def: OpR_RR_pat<V6_vminub, Umin, VecI8, HVI8>; + def: OpR_RR_pat<V6_vmaxub, Umax, VecI8, HVI8>; + def: OpR_RR_pat<V6_vminh, Smin, VecI16, HVI16>; + def: OpR_RR_pat<V6_vmaxh, Smax, VecI16, HVI16>; + def: OpR_RR_pat<V6_vminuh, Umin, VecI16, HVI16>; + def: OpR_RR_pat<V6_vmaxuh, Umax, VecI16, HVI16>; + def: OpR_RR_pat<V6_vminw, Smin, VecI32, HVI32>; + def: OpR_RR_pat<V6_vmaxw, Smax, VecI32, HVI32>; + def: Pat<(vselect HQ8:$Qu, HVI8:$Vs, HVI8:$Vt), (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; def: Pat<(vselect HQ16:$Qu, HVI16:$Vs, HVI16:$Vt), @@ -308,6 +330,20 @@ let Predicates = [UseHVX] in { } let Predicates = [UseHVX] in { + // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), + // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, + // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). + def: Pat<(mul HVI8:$Vs, HVI8:$Vt), + (V6_vshuffeb (HiVec (V6_vmpybv HvxVR:$Vs, HvxVR:$Vt)), + (LoVec (V6_vmpybv HvxVR:$Vs, HvxVR:$Vt)))>; + def: Pat<(mul HVI16:$Vs, HVI16:$Vt), + (V6_vmpyih HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(mul HVI32:$Vs, HVI32:$Vt), + (V6_vmpyiewuh_acc (V6_vmpyieoh HvxVR:$Vs, HvxVR:$Vt), + HvxVR:$Vs, HvxVR:$Vt)>; +} + +let Predicates = [UseHVX] in { def: Pat<(VecPI16 (sext HVI8:$Vs)), (VSxtb $Vs)>; def: Pat<(VecPI32 (sext HVI16:$Vs)), (VSxth $Vs)>; def: Pat<(VecPI16 (zext HVI8:$Vs)), (VZxtb $Vs)>; @@ -364,6 +400,14 @@ let Predicates = [UseHVX] in { (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 16)), (A2_tfrsi 16))>; } + // Take a pair of vectors Vt:Vs and shift them towards LSB by (Rt & HwLen). + def: Pat<(VecI8 (valign HVI8:$Vt, HVI8:$Vs, I32:$Rt)), + (LoVec (V6_valignb HvxVR:$Vt, HvxVR:$Vs, I32:$Rt))>; + def: Pat<(VecI16 (valign HVI16:$Vt, HVI16:$Vs, I32:$Rt)), + (LoVec (V6_valignb HvxVR:$Vt, HvxVR:$Vs, I32:$Rt))>; + def: Pat<(VecI32 (valign HVI32:$Vt, HVI32:$Vs, I32:$Rt)), + (LoVec (V6_valignb HvxVR:$Vt, HvxVR:$Vs, I32:$Rt))>; + def: Pat<(HexagonVASL HVI8:$Vs, I32:$Rt), (V6_vpackeb (V6_vaslh (HiVec (VZxtb HvxVR:$Vs)), I32:$Rt), (V6_vaslh (LoVec (VZxtb HvxVR:$Vs)), I32:$Rt))>; @@ -393,10 +437,43 @@ let Predicates = [UseHVX] in { def: Pat<(srl HVI16:$Vs, HVI16:$Vt), (V6_vlsrhv HvxVR:$Vs, HvxVR:$Vt)>; def: Pat<(srl HVI32:$Vs, HVI32:$Vt), (V6_vlsrwv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecI16 (bswap HVI16:$Vs)), - (V6_vdelta HvxVR:$Vs, (V6_lvsplatw (A2_tfrsi 0x01010101)))>; - def: Pat<(VecI32 (bswap HVI32:$Vs)), - (V6_vdelta HvxVR:$Vs, (V6_lvsplatw (A2_tfrsi 0x03030303)))>; + // Vpackl is a pseudo-op that is used when legalizing widened truncates. + // It should never be produced with a register pair in the output, but + // it can happen to have a pair as an input. + def: Pat<(VecI8 (vpackl HVI16:$Vs)), (V6_vdealb HvxVR:$Vs)>; + def: Pat<(VecI8 (vpackl HVI32:$Vs)), (V6_vdealb4w (IMPLICIT_DEF), HvxVR:$Vs)>; + def: Pat<(VecI16 (vpackl HVI32:$Vs)), (V6_vdealh HvxVR:$Vs)>; + def: Pat<(VecI8 (vpackl HWI16:$Vs)), (V6_vpackeb (HiVec $Vs), (LoVec $Vs))>; + def: Pat<(VecI8 (vpackl HWI32:$Vs)), + (V6_vpackeb (IMPLICIT_DEF), (V6_vpackeh (HiVec $Vs), (LoVec $Vs)))>; + def: Pat<(VecI16 (vpackl HWI32:$Vs)), (V6_vpackeh (HiVec $Vs), (LoVec $Vs))>; + + def: Pat<(VecI16 (vunpack HVI8:$Vs)), (LoVec (VSxtb $Vs))>; + def: Pat<(VecI32 (vunpack HVI8:$Vs)), (LoVec (VSxth (LoVec (VSxtb $Vs))))>; + def: Pat<(VecI32 (vunpack HVI16:$Vs)), (LoVec (VSxth $Vs))>; + def: Pat<(VecPI16 (vunpack HVI8:$Vs)), (VSxtb $Vs)>; + def: Pat<(VecPI32 (vunpack HVI8:$Vs)), (VSxth (LoVec (VSxtb $Vs)))>; + def: Pat<(VecPI32 (vunpack HVI32:$Vs)), (VSxth $Vs)>; + + def: Pat<(VecI16 (vunpacku HVI8:$Vs)), (LoVec (VZxtb $Vs))>; + def: Pat<(VecI32 (vunpacku HVI8:$Vs)), (LoVec (VZxth (LoVec (VZxtb $Vs))))>; + def: Pat<(VecI32 (vunpacku HVI16:$Vs)), (LoVec (VZxth $Vs))>; + def: Pat<(VecPI16 (vunpacku HVI8:$Vs)), (VZxtb $Vs)>; + def: Pat<(VecPI32 (vunpacku HVI8:$Vs)), (VZxth (LoVec (VZxtb $Vs)))>; + def: Pat<(VecPI32 (vunpacku HVI32:$Vs)), (VZxth $Vs)>; + + let Predicates = [UseHVX,UseHVXV60] in { + def: Pat<(VecI16 (bswap HVI16:$Vs)), + (V6_vdelta HvxVR:$Vs, (V60splatib (i32 0x01)))>; + def: Pat<(VecI32 (bswap HVI32:$Vs)), + (V6_vdelta HvxVR:$Vs, (V60splatib (i32 0x03)))>; + } + let Predicates = [UseHVX,UseHVXV62], AddedComplexity = 10 in { + def: Pat<(VecI16 (bswap HVI16:$Vs)), + (V6_vdelta HvxVR:$Vs, (V62splatib (i32 0x01)))>; + def: Pat<(VecI32 (bswap HVI32:$Vs)), + (V6_vdelta HvxVR:$Vs, (V62splatib (i32 0x03)))>; + } def: Pat<(VecI8 (ctpop HVI8:$Vs)), (V6_vpackeb (V6_vpopcounth (HiVec (V6_vunpackub HvxVR:$Vs))), @@ -406,10 +483,17 @@ let Predicates = [UseHVX] in { (V6_vaddw (LoVec (V6_vzh (V6_vpopcounth HvxVR:$Vs))), (HiVec (V6_vzh (V6_vpopcounth HvxVR:$Vs))))>; + let Predicates = [UseHVX,UseHVXV60] in def: Pat<(VecI8 (ctlz HVI8:$Vs)), (V6_vsubb (V6_vpackeb (V6_vcl0h (HiVec (V6_vunpackub HvxVR:$Vs))), (V6_vcl0h (LoVec (V6_vunpackub HvxVR:$Vs)))), - (V6_lvsplatw (A2_tfrsi 0x08080808)))>; + (V60splatib (i32 0x08)))>; + let Predicates = [UseHVX,UseHVXV62], AddedComplexity = 10 in + def: Pat<(VecI8 (ctlz HVI8:$Vs)), + (V6_vsubb (V6_vpackeb (V6_vcl0h (HiVec (V6_vunpackub HvxVR:$Vs))), + (V6_vcl0h (LoVec (V6_vunpackub HvxVR:$Vs)))), + (V62splatib (i32 0x08)))>; + def: Pat<(VecI16 (ctlz HVI16:$Vs)), (V6_vcl0h HvxVR:$Vs)>; def: Pat<(VecI32 (ctlz HVI32:$Vs)), (V6_vcl0w HvxVR:$Vs)>; } diff --git a/llvm/lib/Target/Hexagon/HexagonPeephole.cpp b/llvm/lib/Target/Hexagon/HexagonPeephole.cpp index d0b02f035d1e..fc31139e13ce 100644 --- a/llvm/lib/Target/Hexagon/HexagonPeephole.cpp +++ b/llvm/lib/Target/Hexagon/HexagonPeephole.cpp @@ -139,8 +139,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) { Register DstReg = Dst.getReg(); Register SrcReg = Src.getReg(); // Just handle virtual registers. - if (Register::isVirtualRegister(DstReg) && - Register::isVirtualRegister(SrcReg)) { + if (DstReg.isVirtual() && SrcReg.isVirtual()) { // Map the following: // %170 = SXTW %166 // PeepholeMap[170] = %166 @@ -188,8 +187,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) { Register DstReg = Dst.getReg(); Register SrcReg = Src.getReg(); // Just handle virtual registers. - if (Register::isVirtualRegister(DstReg) && - Register::isVirtualRegister(SrcReg)) { + if (DstReg.isVirtual() && SrcReg.isVirtual()) { // Map the following: // %170 = NOT_xx %166 // PeepholeMap[170] = %166 @@ -210,8 +208,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) { Register DstReg = Dst.getReg(); Register SrcReg = Src.getReg(); - if (Register::isVirtualRegister(DstReg) && - Register::isVirtualRegister(SrcReg)) { + if (DstReg.isVirtual() && SrcReg.isVirtual()) { // Try to find in the map. if (unsigned PeepholeSrc = PeepholeMap.lookup(SrcReg)) { // Change the 1st operand. @@ -242,7 +239,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) { if (RC0->getID() == Hexagon::PredRegsRegClassID) { // Handle instructions that have a prediate register in op0 // (most cases of predicable instructions). - if (Register::isVirtualRegister(Reg0)) { + if (Reg0.isVirtual()) { // Try to find in the map. if (unsigned PeepholeSrc = PeepholeMap.lookup(Reg0)) { // Change the 1st operand and, flip the opcode. diff --git a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp index 52f247977094..5ece577e8285 100644 --- a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp @@ -207,7 +207,7 @@ void HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, int FI = MI.getOperand(FIOp).getIndex(); // Select the base pointer (BP) and calculate the actual offset from BP // to the beginning of the object at index FI. - int Offset = HFI.getFrameIndexReference(MF, FI, BP); + int Offset = HFI.getFrameIndexReference(MF, FI, BP).getFixed(); // Add the offset from the instruction. int RealOffset = Offset + MI.getOperand(FIOp+1).getImm(); bool IsKill = false; diff --git a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp index b45d871e04d6..c8c66ebb69cd 100644 --- a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp +++ b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp @@ -97,7 +97,7 @@ namespace { bool isFixedInstr(const MachineInstr *MI) const; void partitionRegisters(UUSetMap &P2Rs); int32_t profit(const MachineInstr *MI) const; - int32_t profit(unsigned Reg) const; + int32_t profit(Register Reg) const; bool isProfitable(const USet &Part, LoopRegMap &IRM) const; void collectIndRegsForLoop(const MachineLoop *L, USet &Rs); @@ -211,7 +211,7 @@ bool HexagonSplitDoubleRegs::isFixedInstr(const MachineInstr *MI) const { if (!Op.isReg()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) return true; } return false; @@ -259,7 +259,7 @@ void HexagonSplitDoubleRegs::partitionRegisters(UUSetMap &P2Rs) { if (&MO == &Op || !MO.isReg() || MO.getSubReg()) continue; Register T = MO.getReg(); - if (!Register::isVirtualRegister(T)) { + if (!T.isVirtual()) { FixedRegs.set(x); continue; } @@ -399,8 +399,8 @@ int32_t HexagonSplitDoubleRegs::profit(const MachineInstr *MI) const { return 0; } -int32_t HexagonSplitDoubleRegs::profit(unsigned Reg) const { - assert(Register::isVirtualRegister(Reg)); +int32_t HexagonSplitDoubleRegs::profit(Register Reg) const { + assert(Reg.isVirtual()); const MachineInstr *DefI = MRI->getVRegDef(Reg); switch (DefI->getOpcode()) { @@ -574,12 +574,9 @@ void HexagonSplitDoubleRegs::collectIndRegs(LoopRegMap &IRM) { LoopVector WorkQ; - for (auto I : *MLI) - WorkQ.push_back(I); - for (unsigned i = 0; i < WorkQ.size(); ++i) { - for (auto I : *WorkQ[i]) - WorkQ.push_back(I); - } + append_range(WorkQ, *MLI); + for (unsigned i = 0; i < WorkQ.size(); ++i) + append_range(WorkQ, *WorkQ[i]); USet Rs; for (unsigned i = 0, n = WorkQ.size(); i < n; ++i) { @@ -605,7 +602,7 @@ void HexagonSplitDoubleRegs::createHalfInstr(unsigned Opc, MachineInstr *MI, // For register operands, set the subregister. Register R = Op.getReg(); unsigned SR = Op.getSubReg(); - bool isVirtReg = Register::isVirtualRegister(R); + bool isVirtReg = R.isVirtual(); bool isKill = Op.isKill(); if (isVirtReg && MRI->getRegClass(R) == DoubleRC) { isKill = false; @@ -1106,7 +1103,7 @@ void HexagonSplitDoubleRegs::collapseRegPairs(MachineInstr *MI, if (!Op.isReg() || !Op.isUse()) continue; Register R = Op.getReg(); - if (!Register::isVirtualRegister(R)) + if (!R.isVirtual()) continue; if (MRI->getRegClass(R) != DoubleRC || Op.getSubReg()) continue; diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp index 2b7e1bcba9a3..87b1c43961d7 100644 --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// +#include "HexagonSubtarget.h" #include "Hexagon.h" #include "HexagonInstrInfo.h" #include "HexagonRegisterInfo.h" -#include "HexagonSubtarget.h" #include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" @@ -26,6 +26,7 @@ #include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Target/TargetMachine.h" #include <algorithm> #include <cassert> #include <map> @@ -38,7 +39,6 @@ using namespace llvm; #define GET_SUBTARGETINFO_TARGET_DESC #include "HexagonGenSubtargetInfo.inc" - static cl::opt<bool> EnableBSBSched("enable-bsb-sched", cl::Hidden, cl::ZeroOrMore, cl::init(true)); @@ -77,7 +77,8 @@ static cl::opt<bool> EnableCheckBankConflict("hexagon-check-bank-conflict", HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU, StringRef FS, const TargetMachine &TM) - : HexagonGenSubtargetInfo(TT, CPU, FS), OptLevel(TM.getOptLevel()), + : HexagonGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), + OptLevel(TM.getOptLevel()), CPUString(std::string(Hexagon_MC::selectHexagonCPU(CPU))), TargetTriple(TT), InstrInfo(initializeSubtargetDependencies(CPU, FS)), RegInfo(getHwMode()), TLInfo(TM, *this), @@ -104,7 +105,7 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { UseBSBScheduling = hasV60Ops() && EnableBSBSched; - ParseSubtargetFeatures(CPUString, FS); + ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FS); if (OverrideLongCalls.getPosition()) UseLongCalls = OverrideLongCalls; @@ -124,6 +125,76 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { return *this; } +bool HexagonSubtarget::isHVXElementType(MVT Ty, bool IncludeBool) const { + if (!useHVXOps()) + return false; + if (Ty.isVector()) + Ty = Ty.getVectorElementType(); + if (IncludeBool && Ty == MVT::i1) + return true; + ArrayRef<MVT> ElemTypes = getHVXElementTypes(); + return llvm::is_contained(ElemTypes, Ty); +} + +bool HexagonSubtarget::isHVXVectorType(MVT VecTy, bool IncludeBool) const { + if (!VecTy.isVector() || !useHVXOps() || VecTy.isScalableVector()) + return false; + MVT ElemTy = VecTy.getVectorElementType(); + if (!IncludeBool && ElemTy == MVT::i1) + return false; + + unsigned HwLen = getVectorLength(); + unsigned NumElems = VecTy.getVectorNumElements(); + ArrayRef<MVT> ElemTypes = getHVXElementTypes(); + + if (IncludeBool && ElemTy == MVT::i1) { + // Boolean HVX vector types are formed from regular HVX vector types + // by replacing the element type with i1. + for (MVT T : ElemTypes) + if (NumElems * T.getSizeInBits() == 8 * HwLen) + return true; + return false; + } + + unsigned VecWidth = VecTy.getSizeInBits(); + if (VecWidth != 8 * HwLen && VecWidth != 16 * HwLen) + return false; + return llvm::is_contained(ElemTypes, ElemTy); +} + +bool HexagonSubtarget::isTypeForHVX(Type *VecTy, bool IncludeBool) const { + if (!VecTy->isVectorTy() || isa<ScalableVectorType>(VecTy)) + return false; + // Avoid types like <2 x i32*>. + if (!cast<VectorType>(VecTy)->getElementType()->isIntegerTy()) + return false; + // The given type may be something like <17 x i32>, which is not MVT, + // but can be represented as (non-simple) EVT. + EVT Ty = EVT::getEVT(VecTy, /*HandleUnknown*/false); + if (Ty.getSizeInBits() <= 64 || !Ty.getVectorElementType().isSimple()) + return false; + + auto isHvxTy = [this, IncludeBool](MVT SimpleTy) { + if (isHVXVectorType(SimpleTy, IncludeBool)) + return true; + auto Action = getTargetLowering()->getPreferredVectorAction(SimpleTy); + return Action == TargetLoweringBase::TypeWidenVector; + }; + + // Round up EVT to have power-of-2 elements, and keep checking if it + // qualifies for HVX, dividing it in half after each step. + MVT ElemTy = Ty.getVectorElementType().getSimpleVT(); + unsigned VecLen = PowerOf2Ceil(Ty.getVectorNumElements()); + while (ElemTy.getSizeInBits() * VecLen > 64) { + MVT SimpleTy = MVT::getVectorVT(ElemTy, VecLen); + if (SimpleTy.isValid() && isHvxTy(SimpleTy)) + return true; + VecLen /= 2; + } + + return false; +} + void HexagonSubtarget::UsrOverflowMutation::apply(ScheduleDAGInstrs *DAG) { for (SUnit &SU : DAG->SUnits) { if (!SU.isInstr()) @@ -420,14 +491,14 @@ void HexagonSubtarget::restoreLatency(SUnit *Src, SUnit *Dst) const { for (auto &I : Src->Succs) { if (!I.isAssignedRegDep() || I.getSUnit() != Dst) continue; - unsigned DepR = I.getReg(); + Register DepR = I.getReg(); int DefIdx = -1; for (unsigned OpNum = 0; OpNum < SrcI->getNumOperands(); OpNum++) { const MachineOperand &MO = SrcI->getOperand(OpNum); bool IsSameOrSubReg = false; if (MO.isReg()) { - unsigned MOReg = MO.getReg(); - if (Register::isVirtualRegister(DepR)) { + Register MOReg = MO.getReg(); + if (DepR.isVirtual()) { IsSameOrSubReg = (MOReg == DepR); } else { IsSameOrSubReg = getRegisterInfo()->isSubRegisterEq(DepR, MOReg); @@ -456,7 +527,7 @@ void HexagonSubtarget::restoreLatency(SUnit *Src, SUnit *Dst) const { // Update the latency of opposite edge too. T.setSUnit(Src); - auto F = std::find(Dst->Preds.begin(), Dst->Preds.end(), T); + auto F = find(Dst->Preds, T); assert(F != Dst->Preds.end()); F->setLatency(I.getLatency()); } @@ -473,7 +544,7 @@ void HexagonSubtarget::changeLatency(SUnit *Src, SUnit *Dst, unsigned Lat) // Update the latency of opposite edge too. T.setSUnit(Src); - auto F = std::find(Dst->Preds.begin(), Dst->Preds.end(), T); + auto F = find(Dst->Preds, T); assert(F != Dst->Preds.end()); F->setLatency(Lat); } diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h index de4f245519e4..7b7fb8d04f47 100644 --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h @@ -135,7 +135,7 @@ public: /// ParseSubtargetFeatures - Parses features string setting specified /// subtarget options. Definition of function is auto generated by tblgen. - void ParseSubtargetFeatures(StringRef CPU, StringRef FS); + void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); bool hasV5Ops() const { return getHexagonArchVersion() >= Hexagon::ArchEnum::V5; @@ -275,31 +275,9 @@ public: return makeArrayRef(Types); } - bool isHVXVectorType(MVT VecTy, bool IncludeBool = false) const { - if (!VecTy.isVector() || !useHVXOps() || VecTy.isScalableVector()) - return false; - MVT ElemTy = VecTy.getVectorElementType(); - if (!IncludeBool && ElemTy == MVT::i1) - return false; - - unsigned HwLen = getVectorLength(); - unsigned NumElems = VecTy.getVectorNumElements(); - ArrayRef<MVT> ElemTypes = getHVXElementTypes(); - - if (IncludeBool && ElemTy == MVT::i1) { - // Boolean HVX vector types are formed from regular HVX vector types - // by replacing the element type with i1. - for (MVT T : ElemTypes) - if (NumElems * T.getSizeInBits() == 8*HwLen) - return true; - return false; - } - - unsigned VecWidth = VecTy.getSizeInBits(); - if (VecWidth != 8*HwLen && VecWidth != 16*HwLen) - return false; - return llvm::any_of(ElemTypes, [ElemTy] (MVT T) { return ElemTy == T; }); - } + bool isHVXElementType(MVT Ty, bool IncludeBool = false) const; + bool isHVXVectorType(MVT VecTy, bool IncludeBool = false) const; + bool isTypeForHVX(Type *VecTy, bool IncludeBool = false) const; unsigned getTypeAlignment(MVT Ty) const { if (isHVXVectorType(Ty, true)) diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 3fe42ea13f51..9195bb3dc725 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -13,14 +13,17 @@ #include "HexagonTargetMachine.h" #include "Hexagon.h" #include "HexagonISelLowering.h" +#include "HexagonLoopIdiomRecognition.h" #include "HexagonMachineScheduler.h" #include "HexagonTargetObjectFile.h" #include "HexagonTargetTransformInfo.h" +#include "HexagonVectorLoopCarriedReuse.h" #include "TargetInfo/HexagonTargetInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" @@ -97,10 +100,17 @@ static cl::opt<bool> EnableVectorPrint("enable-hexagon-vector-print", static cl::opt<bool> EnableVExtractOpt("hexagon-opt-vextract", cl::Hidden, cl::ZeroOrMore, cl::init(true), cl::desc("Enable vextract optimization")); +static cl::opt<bool> EnableVectorCombine("hexagon-vector-combine", cl::Hidden, + cl::ZeroOrMore, cl::init(true), cl::desc("Enable HVX vector combining")); + static cl::opt<bool> EnableInitialCFGCleanup("hexagon-initial-cfg-cleanup", cl::Hidden, cl::ZeroOrMore, cl::init(true), cl::desc("Simplify the CFG after atomic expansion pass")); +static cl::opt<bool> EnableInstSimplify("hexagon-instsimplify", cl::Hidden, + cl::ZeroOrMore, cl::init(true), + cl::desc("Enable instsimplify")); + /// HexagonTargetMachineModule - Note that this is used on hosts that /// cannot link in a library unless there are references into the /// library. In particular, it seems that it is not possible to get @@ -132,16 +142,17 @@ namespace llvm { void initializeHexagonExpandCondsetsPass(PassRegistry&); void initializeHexagonGenMuxPass(PassRegistry&); void initializeHexagonHardwareLoopsPass(PassRegistry&); - void initializeHexagonLoopIdiomRecognizePass(PassRegistry&); - void initializeHexagonVectorLoopCarriedReusePass(PassRegistry&); + void initializeHexagonLoopIdiomRecognizeLegacyPassPass(PassRegistry &); void initializeHexagonNewValueJumpPass(PassRegistry&); void initializeHexagonOptAddrModePass(PassRegistry&); void initializeHexagonPacketizerPass(PassRegistry&); void initializeHexagonRDFOptPass(PassRegistry&); void initializeHexagonSplitDoubleRegsPass(PassRegistry&); + void initializeHexagonVectorCombineLegacyPass(PassRegistry&); + void initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PassRegistry &); void initializeHexagonVExtractPass(PassRegistry&); Pass *createHexagonLoopIdiomPass(); - Pass *createHexagonVectorLoopCarriedReusePass(); + Pass *createHexagonVectorLoopCarriedReuseLegacyPass(); FunctionPass *createHexagonBitSimplify(); FunctionPass *createHexagonBranchRelaxation(); @@ -162,22 +173,21 @@ namespace llvm { CodeGenOpt::Level OptLevel); FunctionPass *createHexagonLoopRescheduling(); FunctionPass *createHexagonNewValueJump(); - FunctionPass *createHexagonOptimizeSZextends(); FunctionPass *createHexagonOptAddrMode(); + FunctionPass *createHexagonOptimizeSZextends(); FunctionPass *createHexagonPacketizer(bool Minimal); FunctionPass *createHexagonPeephole(); FunctionPass *createHexagonRDFOpt(); FunctionPass *createHexagonSplitConst32AndConst64(); FunctionPass *createHexagonSplitDoubleRegs(); FunctionPass *createHexagonStoreWidening(); + FunctionPass *createHexagonVectorCombineLegacyPass(); FunctionPass *createHexagonVectorPrint(); FunctionPass *createHexagonVExtract(); } // end namespace llvm; static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) { - if (!RM.hasValue()) - return Reloc::Static; - return *RM; + return RM.getValueOr(Reloc::Static); } extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonTarget() { @@ -191,13 +201,14 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonTarget() { initializeHexagonEarlyIfConversionPass(PR); initializeHexagonGenMuxPass(PR); initializeHexagonHardwareLoopsPass(PR); - initializeHexagonLoopIdiomRecognizePass(PR); - initializeHexagonVectorLoopCarriedReusePass(PR); + initializeHexagonLoopIdiomRecognizeLegacyPassPass(PR); initializeHexagonNewValueJumpPass(PR); initializeHexagonOptAddrModePass(PR); initializeHexagonPacketizerPass(PR); initializeHexagonRDFOptPass(PR); initializeHexagonSplitDoubleRegsPass(PR); + initializeHexagonVectorCombineLegacyPass(PR); + initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PR); initializeHexagonVExtractPass(PR); } @@ -231,12 +242,10 @@ HexagonTargetMachine::getSubtargetImpl(const Function &F) const { Attribute FSAttr = FnAttrs.getAttribute(AttributeList::FunctionIndex, "target-features"); - std::string CPU = !CPUAttr.hasAttribute(Attribute::None) - ? CPUAttr.getValueAsString().str() - : TargetCPU; - std::string FS = !FSAttr.hasAttribute(Attribute::None) - ? FSAttr.getValueAsString().str() - : TargetFS; + std::string CPU = + CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU; + std::string FS = + FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS; // Append the preexisting target features last, so that +mattr overrides // the "unsafe-fp-math" function attribute. // Creating a separate target feature is not strictly necessary, it only @@ -264,10 +273,22 @@ void HexagonTargetMachine::adjustPassManager(PassManagerBuilder &PMB) { PM.add(createHexagonLoopIdiomPass()); }); PMB.addExtension( - PassManagerBuilder::EP_LoopOptimizerEnd, - [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { - PM.add(createHexagonVectorLoopCarriedReusePass()); - }); + PassManagerBuilder::EP_LoopOptimizerEnd, + [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { + PM.add(createHexagonVectorLoopCarriedReuseLegacyPass()); + }); +} + +void HexagonTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, + bool DebugPassManager) { + PB.registerLateLoopOptimizationsEPCallback( + [=](LoopPassManager &LPM, PassBuilder::OptimizationLevel Level) { + LPM.addPass(HexagonLoopIdiomRecognitionPass()); + }); + PB.registerLoopOptimizerEndEPCallback( + [=](LoopPassManager &LPM, PassBuilder::OptimizationLevel Level) { + LPM.addPass(HexagonVectorLoopCarriedReusePass()); + }); } TargetTransformInfo @@ -312,7 +333,8 @@ void HexagonPassConfig::addIRPasses() { bool NoOpt = (getOptLevel() == CodeGenOpt::None); if (!NoOpt) { - addPass(createConstantPropagationPass()); + if (EnableInstSimplify) + addPass(createInstSimplifyLegacyPass()); addPass(createDeadCodeEliminationPass()); } @@ -320,9 +342,16 @@ void HexagonPassConfig::addIRPasses() { if (!NoOpt) { if (EnableInitialCFGCleanup) - addPass(createCFGSimplificationPass(1, true, true, false, true)); + addPass(createCFGSimplificationPass(SimplifyCFGOptions() + .forwardSwitchCondToPhi(true) + .convertSwitchToLookupTable(true) + .needCanonicalLoops(false) + .hoistCommonInsts(true) + .sinkCommonInsts(true))); if (EnableLoopPrefetch) addPass(createLoopDataPrefetchPass()); + if (EnableVectorCombine) + addPass(createHexagonVectorCombineLegacyPass()); if (EnableCommGEP) addPass(createHexagonCommonGEP()); // Replace certain combinations of shifts and ands with extracts. diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h index 7ee4474e90e3..fa174128f708 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h @@ -37,6 +37,8 @@ public: static unsigned getModuleMatchQuality(const Module &M); void adjustPassManager(PassManagerBuilder &PMB) override; + void registerPassBuilderCallbacks(PassBuilder &PB, + bool DebugPassManager) override; TargetPassConfig *createPassConfig(PassManagerBase &PM) override; TargetTransformInfo getTargetTransformInfo(const Function &F) override; diff --git a/llvm/lib/Target/Hexagon/HexagonTargetObjectFile.cpp b/llvm/lib/Target/Hexagon/HexagonTargetObjectFile.cpp index cfc8ed813c92..595cf94e3f1d 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetObjectFile.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetObjectFile.cpp @@ -331,6 +331,7 @@ unsigned HexagonTargetObjectFile::getSmallestAddressableSize(const Type *Ty, case Type::LabelTyID: case Type::MetadataTyID: case Type::X86_MMXTyID: + case Type::X86_AMXTyID: case Type::TokenTyID: return 0; } diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp index 80c8736cb74a..1cefa6a04640 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/User.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/Utils/LoopPeel.h" #include "llvm/Transforms/Utils/UnrollLoop.h" using namespace llvm; @@ -34,6 +35,9 @@ static cl::opt<bool> EmitLookupTables("hexagon-emit-lookup-tables", cl::init(true), cl::Hidden, cl::desc("Control lookup table emission on Hexagon target")); +static cl::opt<bool> HexagonMaskedVMem("hexagon-masked-vmem", cl::init(true), + cl::Hidden, cl::desc("Enable masked loads/stores for HVX")); + // Constant "cost factor" to make floating point operations more expensive // in terms of vectorization cost. This isn't the best way, but it should // do. Ultimately, the cost should use cycles. @@ -43,22 +47,6 @@ bool HexagonTTIImpl::useHVX() const { return ST.useHVXOps() && HexagonAutoHVX; } -bool HexagonTTIImpl::isTypeForHVX(Type *VecTy) const { - assert(VecTy->isVectorTy()); - if (isa<ScalableVectorType>(VecTy)) - return false; - // Avoid types like <2 x i32*>. - if (!cast<VectorType>(VecTy)->getElementType()->isIntegerTy()) - return false; - EVT VecVT = EVT::getEVT(VecTy); - if (!VecVT.isSimple() || VecVT.getSizeInBits() <= 64) - return false; - if (ST.isHVXVectorType(VecVT.getSimpleVT())) - return true; - auto Action = TLI.getPreferredVectorAction(VecVT.getSimpleVT()); - return Action == TargetLoweringBase::TypeWidenVector; -} - unsigned HexagonTTIImpl::getTypeNumElements(Type *Ty) const { if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) return VTy->getNumElements(); @@ -84,7 +72,7 @@ void HexagonTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP) { BaseT::getPeelingPreferences(L, SE, PP); // Only try to peel innermost loops with small runtime trip counts. - if (L && L->empty() && canPeel(L) && + if (L && L->isInnermost() && canPeel(L) && SE.getSmallConstantTripCount(L) == 0 && SE.getSmallConstantMaxTripCount(L) > 0 && SE.getSmallConstantMaxTripCount(L) <= 5) { @@ -105,7 +93,7 @@ unsigned HexagonTTIImpl::getNumberOfRegisters(bool Vector) const { } unsigned HexagonTTIImpl::getMaxInterleaveFactor(unsigned VF) { - return useHVX() ? 2 : 0; + return useHVX() ? 2 : 1; } unsigned HexagonTTIImpl::getRegisterBitWidth(bool Vector) const { @@ -113,7 +101,7 @@ unsigned HexagonTTIImpl::getRegisterBitWidth(bool Vector) const { } unsigned HexagonTTIImpl::getMinVectorRegisterBitWidth() const { - return useHVX() ? ST.getVectorLength()*8 : 0; + return useHVX() ? ST.getVectorLength()*8 : 32; } unsigned HexagonTTIImpl::getMinimumVF(unsigned ElemWidth) const { @@ -168,7 +156,7 @@ unsigned HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, if (Src->isVectorTy()) { VectorType *VecTy = cast<VectorType>(Src); unsigned VecWidth = VecTy->getPrimitiveSizeInBits().getFixedSize(); - if (useHVX() && isTypeForHVX(VecTy)) { + if (useHVX() && ST.isTypeForHVX(VecTy)) { unsigned RegWidth = getRegisterBitWidth(true); assert(RegWidth && "Non-zero vector register width expected"); // Cost of HVX loads. @@ -239,13 +227,16 @@ unsigned HexagonTTIImpl::getInterleavedMemoryOpCost( } unsigned HexagonTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy, TTI::TargetCostKind CostKind, const Instruction *I) { + Type *CondTy, + CmpInst::Predicate VecPred, + TTI::TargetCostKind CostKind, + const Instruction *I) { if (ValTy->isVectorTy() && CostKind == TTI::TCK_RecipThroughput) { std::pair<int, MVT> LT = TLI.getTypeLegalizationCost(DL, ValTy); if (Opcode == Instruction::FCmp) return LT.first + FloatFactor * getTypeNumElements(ValTy); } - return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, CostKind, I); + return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); } unsigned HexagonTTIImpl::getArithmeticInstrCost( @@ -270,7 +261,9 @@ unsigned HexagonTTIImpl::getArithmeticInstrCost( } unsigned HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy, - Type *SrcTy, TTI::TargetCostKind CostKind, const Instruction *I) { + Type *SrcTy, TTI::CastContextHint CCH, + TTI::TargetCostKind CostKind, + const Instruction *I) { if (SrcTy->isFPOrFPVectorTy() || DstTy->isFPOrFPVectorTy()) { unsigned SrcN = SrcTy->isFPOrFPVectorTy() ? getTypeNumElements(SrcTy) : 0; unsigned DstN = DstTy->isFPOrFPVectorTy() ? getTypeNumElements(DstTy) : 0; @@ -305,6 +298,14 @@ unsigned HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, return 1; } +bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/) { + return HexagonMaskedVMem && ST.isTypeForHVX(DataType); +} + +bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/) { + return HexagonMaskedVMem && ST.isTypeForHVX(DataType); +} + /// --- Vector TTI end --- unsigned HexagonTTIImpl::getPrefetchDistance() const { diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index 5fe397486402..835358d3fed0 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -43,7 +43,6 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> { const HexagonTargetLowering *getTLI() const { return &TLI; } bool useHVX() const; - bool isTypeForHVX(Type *VecTy) const; // Returns the number of vector elements of Ty, if Ty is a vector type, // or 1 if Ty is a scalar type. It is incorrect to call this function @@ -134,6 +133,8 @@ public: TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency, bool UseMaskForCond = false, bool UseMaskForGaps = false); unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, + + CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); unsigned getArithmeticInstrCost( @@ -146,14 +147,18 @@ public: ArrayRef<const Value *> Args = ArrayRef<const Value *>(), const Instruction *CxtI = nullptr); unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + TTI::CastContextHint CCH, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) { return 1; } + bool isLegalMaskedStore(Type *DataType, Align Alignment); + bool isLegalMaskedLoad(Type *DataType, Align Alignment); + /// @} int getUserCost(const User *U, ArrayRef<const Value *> Operands, diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp new file mode 100644 index 000000000000..a605fdfcf100 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp @@ -0,0 +1,1487 @@ +//===-- HexagonVectorCombine.cpp ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// HexagonVectorCombine is a utility class implementing a variety of functions +// that assist in vector-based optimizations. +// +// AlignVectors: replace unaligned vector loads and stores with aligned ones. +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsHexagon.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/KnownBits.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" + +#include "HexagonSubtarget.h" +#include "HexagonTargetMachine.h" + +#include <algorithm> +#include <deque> +#include <map> +#include <set> +#include <utility> +#include <vector> + +#define DEBUG_TYPE "hexagon-vc" + +using namespace llvm; + +namespace { +class HexagonVectorCombine { +public: + HexagonVectorCombine(Function &F_, AliasAnalysis &AA_, AssumptionCache &AC_, + DominatorTree &DT_, TargetLibraryInfo &TLI_, + const TargetMachine &TM_) + : F(F_), DL(F.getParent()->getDataLayout()), AA(AA_), AC(AC_), DT(DT_), + TLI(TLI_), + HST(static_cast<const HexagonSubtarget &>(*TM_.getSubtargetImpl(F))) {} + + bool run(); + + // Common integer type. + IntegerType *getIntTy() const; + // Byte type: either scalar (when Length = 0), or vector with given + // element count. + Type *getByteTy(int ElemCount = 0) const; + // Boolean type: either scalar (when Length = 0), or vector with given + // element count. + Type *getBoolTy(int ElemCount = 0) const; + // Create a ConstantInt of type returned by getIntTy with the value Val. + ConstantInt *getConstInt(int Val) const; + // Get the integer value of V, if it exists. + Optional<APInt> getIntValue(const Value *Val) const; + // Is V a constant 0, or a vector of 0s? + bool isZero(const Value *Val) const; + // Is V an undef value? + bool isUndef(const Value *Val) const; + + int getSizeOf(const Value *Val) const; + int getSizeOf(const Type *Ty) const; + int getTypeAlignment(Type *Ty) const; + + VectorType *getByteVectorTy(int ScLen) const; + Constant *getNullValue(Type *Ty) const; + Constant *getFullValue(Type *Ty) const; + + Value *insertb(IRBuilder<> &Builder, Value *Dest, Value *Src, int Start, + int Length, int Where) const; + Value *vlalignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, Value *Amt) const; + Value *vralignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, Value *Amt) const; + Value *concat(IRBuilder<> &Builder, ArrayRef<Value *> Vecs) const; + Value *vresize(IRBuilder<> &Builder, Value *Val, int NewSize, + Value *Pad) const; + Value *rescale(IRBuilder<> &Builder, Value *Mask, Type *FromTy, + Type *ToTy) const; + Value *vlsb(IRBuilder<> &Builder, Value *Val) const; + Value *vbytes(IRBuilder<> &Builder, Value *Val) const; + + Value *createHvxIntrinsic(IRBuilder<> &Builder, Intrinsic::ID IntID, + Type *RetTy, ArrayRef<Value *> Args) const; + + Optional<int> calculatePointerDifference(Value *Ptr0, Value *Ptr1) const; + + template <typename T = std::vector<Instruction *>> + bool isSafeToMoveBeforeInBB(const Instruction &In, + BasicBlock::const_iterator To, + const T &Ignore = {}) const; + + Function &F; + const DataLayout &DL; + AliasAnalysis &AA; + AssumptionCache &AC; + DominatorTree &DT; + TargetLibraryInfo &TLI; + const HexagonSubtarget &HST; + +private: +#ifndef NDEBUG + // These two functions are only used for assertions at the moment. + bool isByteVecTy(Type *Ty) const; + bool isSectorTy(Type *Ty) const; +#endif + Value *getElementRange(IRBuilder<> &Builder, Value *Lo, Value *Hi, int Start, + int Length) const; +}; + +class AlignVectors { +public: + AlignVectors(HexagonVectorCombine &HVC_) : HVC(HVC_) {} + + bool run(); + +private: + using InstList = std::vector<Instruction *>; + + struct Segment { + void *Data; + int Start; + int Size; + }; + + struct AddrInfo { + AddrInfo(const AddrInfo &) = default; + AddrInfo(const HexagonVectorCombine &HVC, Instruction *I, Value *A, Type *T, + Align H) + : Inst(I), Addr(A), ValTy(T), HaveAlign(H), + NeedAlign(HVC.getTypeAlignment(ValTy)) {} + + // XXX: add Size member? + Instruction *Inst; + Value *Addr; + Type *ValTy; + Align HaveAlign; + Align NeedAlign; + int Offset = 0; // Offset (in bytes) from the first member of the + // containing AddrList. + }; + using AddrList = std::vector<AddrInfo>; + + struct InstrLess { + bool operator()(const Instruction *A, const Instruction *B) const { + return A->comesBefore(B); + } + }; + using DepList = std::set<Instruction *, InstrLess>; + + struct MoveGroup { + MoveGroup(const AddrInfo &AI, Instruction *B, bool Hvx, bool Load) + : Base(B), Main{AI.Inst}, IsHvx(Hvx), IsLoad(Load) {} + Instruction *Base; // Base instruction of the parent address group. + InstList Main; // Main group of instructions. + InstList Deps; // List of dependencies. + bool IsHvx; // Is this group of HVX instructions? + bool IsLoad; // Is this a load group? + }; + using MoveList = std::vector<MoveGroup>; + + struct ByteSpan { + struct Segment { + Segment(Value *Val, int Begin, int Len) + : Val(Val), Start(Begin), Size(Len) {} + Segment(const Segment &Seg) = default; + Value *Val; + int Start; + int Size; + }; + + struct Block { + Block(Value *Val, int Len, int Pos) : Seg(Val, 0, Len), Pos(Pos) {} + Block(Value *Val, int Off, int Len, int Pos) + : Seg(Val, Off, Len), Pos(Pos) {} + Block(const Block &Blk) = default; + Segment Seg; + int Pos; + }; + + int extent() const; + ByteSpan section(int Start, int Length) const; + ByteSpan &shift(int Offset); + + int size() const { return Blocks.size(); } + Block &operator[](int i) { return Blocks[i]; } + + std::vector<Block> Blocks; + + using iterator = decltype(Blocks)::iterator; + iterator begin() { return Blocks.begin(); } + iterator end() { return Blocks.end(); } + using const_iterator = decltype(Blocks)::const_iterator; + const_iterator begin() const { return Blocks.begin(); } + const_iterator end() const { return Blocks.end(); } + }; + + Align getAlignFromValue(const Value *V) const; + Optional<MemoryLocation> getLocation(const Instruction &In) const; + Optional<AddrInfo> getAddrInfo(Instruction &In) const; + bool isHvx(const AddrInfo &AI) const; + + Value *getPayload(Value *Val) const; + Value *getMask(Value *Val) const; + Value *getPassThrough(Value *Val) const; + + Value *createAdjustedPointer(IRBuilder<> &Builder, Value *Ptr, Type *ValTy, + int Adjust) const; + Value *createAlignedPointer(IRBuilder<> &Builder, Value *Ptr, Type *ValTy, + int Alignment) const; + Value *createAlignedLoad(IRBuilder<> &Builder, Type *ValTy, Value *Ptr, + int Alignment, Value *Mask, Value *PassThru) const; + Value *createAlignedStore(IRBuilder<> &Builder, Value *Val, Value *Ptr, + int Alignment, Value *Mask) const; + + bool createAddressGroups(); + MoveList createLoadGroups(const AddrList &Group) const; + MoveList createStoreGroups(const AddrList &Group) const; + bool move(const MoveGroup &Move) const; + bool realignGroup(const MoveGroup &Move) const; + + friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI); + friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG); + friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS); + + std::map<Instruction *, AddrList> AddrGroups; + HexagonVectorCombine &HVC; +}; + +LLVM_ATTRIBUTE_UNUSED +raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::AddrInfo &AI) { + OS << "Inst: " << AI.Inst << " " << *AI.Inst << '\n'; + OS << "Addr: " << *AI.Addr << '\n'; + OS << "Type: " << *AI.ValTy << '\n'; + OS << "HaveAlign: " << AI.HaveAlign.value() << '\n'; + OS << "NeedAlign: " << AI.NeedAlign.value() << '\n'; + OS << "Offset: " << AI.Offset; + return OS; +} + +LLVM_ATTRIBUTE_UNUSED +raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::MoveGroup &MG) { + OS << "Main\n"; + for (Instruction *I : MG.Main) + OS << " " << *I << '\n'; + OS << "Deps\n"; + for (Instruction *I : MG.Deps) + OS << " " << *I << '\n'; + return OS; +} + +LLVM_ATTRIBUTE_UNUSED +raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::ByteSpan &BS) { + OS << "ByteSpan[size=" << BS.size() << ", extent=" << BS.extent() << '\n'; + for (const AlignVectors::ByteSpan::Block &B : BS) { + OS << " @" << B.Pos << " [" << B.Seg.Start << ',' << B.Seg.Size << "] " + << *B.Seg.Val << '\n'; + } + OS << ']'; + return OS; +} + +} // namespace + +namespace { + +template <typename T> T *getIfUnordered(T *MaybeT) { + return MaybeT && MaybeT->isUnordered() ? MaybeT : nullptr; +} +template <typename T> T *isCandidate(Instruction *In) { + return dyn_cast<T>(In); +} +template <> LoadInst *isCandidate<LoadInst>(Instruction *In) { + return getIfUnordered(dyn_cast<LoadInst>(In)); +} +template <> StoreInst *isCandidate<StoreInst>(Instruction *In) { + return getIfUnordered(dyn_cast<StoreInst>(In)); +} + +#if !defined(_MSC_VER) || _MSC_VER >= 1924 +// VS2017 has trouble compiling this: +// error C2976: 'std::map': too few template arguments +template <typename Pred, typename... Ts> +void erase_if(std::map<Ts...> &map, Pred p) +#else +template <typename Pred, typename T, typename U> +void erase_if(std::map<T, U> &map, Pred p) +#endif +{ + for (auto i = map.begin(), e = map.end(); i != e;) { + if (p(*i)) + i = map.erase(i); + else + i = std::next(i); + } +} + +// Forward other erase_ifs to the LLVM implementations. +template <typename Pred, typename T> void erase_if(T &&container, Pred p) { + llvm::erase_if(std::forward<T>(container), p); +} + +} // namespace + +// --- Begin AlignVectors + +auto AlignVectors::ByteSpan::extent() const -> int { + if (size() == 0) + return 0; + int Min = Blocks[0].Pos; + int Max = Blocks[0].Pos + Blocks[0].Seg.Size; + for (int i = 1, e = size(); i != e; ++i) { + Min = std::min(Min, Blocks[i].Pos); + Max = std::max(Max, Blocks[i].Pos + Blocks[i].Seg.Size); + } + return Max - Min; +} + +auto AlignVectors::ByteSpan::section(int Start, int Length) const -> ByteSpan { + ByteSpan Section; + for (const ByteSpan::Block &B : Blocks) { + int L = std::max(B.Pos, Start); // Left end. + int R = std::min(B.Pos + B.Seg.Size, Start + Length); // Right end+1. + if (L < R) { + // How much to chop off the beginning of the segment: + int Off = L > B.Pos ? L - B.Pos : 0; + Section.Blocks.emplace_back(B.Seg.Val, B.Seg.Start + Off, R - L, L); + } + } + return Section; +} + +auto AlignVectors::ByteSpan::shift(int Offset) -> ByteSpan & { + for (Block &B : Blocks) + B.Pos += Offset; + return *this; +} + +auto AlignVectors::getAlignFromValue(const Value *V) const -> Align { + const auto *C = dyn_cast<ConstantInt>(V); + assert(C && "Alignment must be a compile-time constant integer"); + return C->getAlignValue(); +} + +auto AlignVectors::getAddrInfo(Instruction &In) const -> Optional<AddrInfo> { + if (auto *L = isCandidate<LoadInst>(&In)) + return AddrInfo(HVC, L, L->getPointerOperand(), L->getType(), + L->getAlign()); + if (auto *S = isCandidate<StoreInst>(&In)) + return AddrInfo(HVC, S, S->getPointerOperand(), + S->getValueOperand()->getType(), S->getAlign()); + if (auto *II = isCandidate<IntrinsicInst>(&In)) { + Intrinsic::ID ID = II->getIntrinsicID(); + switch (ID) { + case Intrinsic::masked_load: + return AddrInfo(HVC, II, II->getArgOperand(0), II->getType(), + getAlignFromValue(II->getArgOperand(1))); + case Intrinsic::masked_store: + return AddrInfo(HVC, II, II->getArgOperand(1), + II->getArgOperand(0)->getType(), + getAlignFromValue(II->getArgOperand(2))); + } + } + return Optional<AddrInfo>(); +} + +auto AlignVectors::isHvx(const AddrInfo &AI) const -> bool { + return HVC.HST.isTypeForHVX(AI.ValTy); +} + +auto AlignVectors::getPayload(Value *Val) const -> Value * { + if (auto *In = dyn_cast<Instruction>(Val)) { + Intrinsic::ID ID = 0; + if (auto *II = dyn_cast<IntrinsicInst>(In)) + ID = II->getIntrinsicID(); + if (isa<StoreInst>(In) || ID == Intrinsic::masked_store) + return In->getOperand(0); + } + return Val; +} + +auto AlignVectors::getMask(Value *Val) const -> Value * { + if (auto *II = dyn_cast<IntrinsicInst>(Val)) { + switch (II->getIntrinsicID()) { + case Intrinsic::masked_load: + return II->getArgOperand(2); + case Intrinsic::masked_store: + return II->getArgOperand(3); + } + } + + Type *ValTy = getPayload(Val)->getType(); + if (auto *VecTy = dyn_cast<VectorType>(ValTy)) { + int ElemCount = VecTy->getElementCount().getFixedValue(); + return HVC.getFullValue(HVC.getBoolTy(ElemCount)); + } + return HVC.getFullValue(HVC.getBoolTy()); +} + +auto AlignVectors::getPassThrough(Value *Val) const -> Value * { + if (auto *II = dyn_cast<IntrinsicInst>(Val)) { + if (II->getIntrinsicID() == Intrinsic::masked_load) + return II->getArgOperand(3); + } + return UndefValue::get(getPayload(Val)->getType()); +} + +auto AlignVectors::createAdjustedPointer(IRBuilder<> &Builder, Value *Ptr, + Type *ValTy, int Adjust) const + -> Value * { + // The adjustment is in bytes, but if it's a multiple of the type size, + // we don't need to do pointer casts. + Type *ElemTy = cast<PointerType>(Ptr->getType())->getElementType(); + int ElemSize = HVC.getSizeOf(ElemTy); + if (Adjust % ElemSize == 0) { + Value *Tmp0 = Builder.CreateGEP(Ptr, HVC.getConstInt(Adjust / ElemSize)); + return Builder.CreatePointerCast(Tmp0, ValTy->getPointerTo()); + } + + PointerType *CharPtrTy = Type::getInt8PtrTy(HVC.F.getContext()); + Value *Tmp0 = Builder.CreatePointerCast(Ptr, CharPtrTy); + Value *Tmp1 = Builder.CreateGEP(Tmp0, HVC.getConstInt(Adjust)); + return Builder.CreatePointerCast(Tmp1, ValTy->getPointerTo()); +} + +auto AlignVectors::createAlignedPointer(IRBuilder<> &Builder, Value *Ptr, + Type *ValTy, int Alignment) const + -> Value * { + Value *AsInt = Builder.CreatePtrToInt(Ptr, HVC.getIntTy()); + Value *Mask = HVC.getConstInt(-Alignment); + Value *And = Builder.CreateAnd(AsInt, Mask); + return Builder.CreateIntToPtr(And, ValTy->getPointerTo()); +} + +auto AlignVectors::createAlignedLoad(IRBuilder<> &Builder, Type *ValTy, + Value *Ptr, int Alignment, Value *Mask, + Value *PassThru) const -> Value * { + assert(!HVC.isUndef(Mask)); // Should this be allowed? + if (HVC.isZero(Mask)) + return PassThru; + if (Mask == ConstantInt::getTrue(Mask->getType())) + return Builder.CreateAlignedLoad(ValTy, Ptr, Align(Alignment)); + return Builder.CreateMaskedLoad(Ptr, Align(Alignment), Mask, PassThru); +} + +auto AlignVectors::createAlignedStore(IRBuilder<> &Builder, Value *Val, + Value *Ptr, int Alignment, + Value *Mask) const -> Value * { + if (HVC.isZero(Mask) || HVC.isUndef(Val) || HVC.isUndef(Mask)) + return UndefValue::get(Val->getType()); + if (Mask == ConstantInt::getTrue(Mask->getType())) + return Builder.CreateAlignedStore(Val, Ptr, Align(Alignment)); + return Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask); +} + +auto AlignVectors::createAddressGroups() -> bool { + // An address group created here may contain instructions spanning + // multiple basic blocks. + AddrList WorkStack; + + auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> { + for (AddrInfo &W : WorkStack) { + if (auto D = HVC.calculatePointerDifference(AI.Addr, W.Addr)) + return std::make_pair(W.Inst, *D); + } + return std::make_pair(nullptr, 0); + }; + + auto traverseBlock = [&](DomTreeNode *DomN, auto Visit) -> void { + BasicBlock &Block = *DomN->getBlock(); + for (Instruction &I : Block) { + auto AI = this->getAddrInfo(I); // Use this-> for gcc6. + if (!AI) + continue; + auto F = findBaseAndOffset(*AI); + Instruction *GroupInst; + if (Instruction *BI = F.first) { + AI->Offset = F.second; + GroupInst = BI; + } else { + WorkStack.push_back(*AI); + GroupInst = AI->Inst; + } + AddrGroups[GroupInst].push_back(*AI); + } + + for (DomTreeNode *C : DomN->children()) + Visit(C, Visit); + + while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &Block) + WorkStack.pop_back(); + }; + + traverseBlock(HVC.DT.getRootNode(), traverseBlock); + assert(WorkStack.empty()); + + // AddrGroups are formed. + + // Remove groups of size 1. + erase_if(AddrGroups, [](auto &G) { return G.second.size() == 1; }); + // Remove groups that don't use HVX types. + erase_if(AddrGroups, [&](auto &G) { + return !llvm::any_of( + G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(I.ValTy); }); + }); + // Remove groups where everything is properly aligned. + erase_if(AddrGroups, [&](auto &G) { + return llvm::all_of(G.second, + [&](auto &I) { return I.HaveAlign >= I.NeedAlign; }); + }); + + return !AddrGroups.empty(); +} + +auto AlignVectors::createLoadGroups(const AddrList &Group) const -> MoveList { + // Form load groups. + // To avoid complications with moving code across basic blocks, only form + // groups that are contained within a single basic block. + + auto getUpwardDeps = [](Instruction *In, Instruction *Base) { + BasicBlock *Parent = Base->getParent(); + assert(In->getParent() == Parent && + "Base and In should be in the same block"); + assert(Base->comesBefore(In) && "Base should come before In"); + + DepList Deps; + std::deque<Instruction *> WorkQ = {In}; + while (!WorkQ.empty()) { + Instruction *D = WorkQ.front(); + WorkQ.pop_front(); + Deps.insert(D); + for (Value *Op : D->operands()) { + if (auto *I = dyn_cast<Instruction>(Op)) { + if (I->getParent() == Parent && Base->comesBefore(I)) + WorkQ.push_back(I); + } + } + } + return Deps; + }; + + auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) { + assert(!Move.Main.empty() && "Move group should have non-empty Main"); + // Don't mix HVX and non-HVX instructions. + if (Move.IsHvx != isHvx(Info)) + return false; + // Leading instruction in the load group. + Instruction *Base = Move.Main.front(); + if (Base->getParent() != Info.Inst->getParent()) + return false; + + auto isSafeToMoveToBase = [&](const Instruction *I) { + return HVC.isSafeToMoveBeforeInBB(*I, Base->getIterator()); + }; + DepList Deps = getUpwardDeps(Info.Inst, Base); + if (!llvm::all_of(Deps, isSafeToMoveToBase)) + return false; + + // The dependencies will be moved together with the load, so make sure + // that none of them could be moved independently in another group. + Deps.erase(Info.Inst); + auto inAddrMap = [&](Instruction *I) { return AddrGroups.count(I) > 0; }; + if (llvm::any_of(Deps, inAddrMap)) + return false; + Move.Main.push_back(Info.Inst); + llvm::append_range(Move.Deps, Deps); + return true; + }; + + MoveList LoadGroups; + + for (const AddrInfo &Info : Group) { + if (!Info.Inst->mayReadFromMemory()) + continue; + if (LoadGroups.empty() || !tryAddTo(Info, LoadGroups.back())) + LoadGroups.emplace_back(Info, Group.front().Inst, isHvx(Info), true); + } + + // Erase singleton groups. + erase_if(LoadGroups, [](const MoveGroup &G) { return G.Main.size() <= 1; }); + return LoadGroups; +} + +auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList { + // Form store groups. + // To avoid complications with moving code across basic blocks, only form + // groups that are contained within a single basic block. + + auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) { + assert(!Move.Main.empty() && "Move group should have non-empty Main"); + // For stores with return values we'd have to collect downward depenencies. + // There are no such stores that we handle at the moment, so omit that. + assert(Info.Inst->getType()->isVoidTy() && + "Not handling stores with return values"); + // Don't mix HVX and non-HVX instructions. + if (Move.IsHvx != isHvx(Info)) + return false; + // For stores we need to be careful whether it's safe to move them. + // Stores that are otherwise safe to move together may not appear safe + // to move over one another (i.e. isSafeToMoveBefore may return false). + Instruction *Base = Move.Main.front(); + if (Base->getParent() != Info.Inst->getParent()) + return false; + if (!HVC.isSafeToMoveBeforeInBB(*Info.Inst, Base->getIterator(), Move.Main)) + return false; + Move.Main.push_back(Info.Inst); + return true; + }; + + MoveList StoreGroups; + + for (auto I = Group.rbegin(), E = Group.rend(); I != E; ++I) { + const AddrInfo &Info = *I; + if (!Info.Inst->mayWriteToMemory()) + continue; + if (StoreGroups.empty() || !tryAddTo(Info, StoreGroups.back())) + StoreGroups.emplace_back(Info, Group.front().Inst, isHvx(Info), false); + } + + // Erase singleton groups. + erase_if(StoreGroups, [](const MoveGroup &G) { return G.Main.size() <= 1; }); + return StoreGroups; +} + +auto AlignVectors::move(const MoveGroup &Move) const -> bool { + assert(!Move.Main.empty() && "Move group should have non-empty Main"); + Instruction *Where = Move.Main.front(); + + if (Move.IsLoad) { + // Move all deps to before Where, keeping order. + for (Instruction *D : Move.Deps) + D->moveBefore(Where); + // Move all main instructions to after Where, keeping order. + ArrayRef<Instruction *> Main(Move.Main); + for (Instruction *M : Main.drop_front(1)) { + M->moveAfter(Where); + Where = M; + } + } else { + // NOTE: Deps are empty for "store" groups. If they need to be + // non-empty, decide on the order. + assert(Move.Deps.empty()); + // Move all main instructions to before Where, inverting order. + ArrayRef<Instruction *> Main(Move.Main); + for (Instruction *M : Main.drop_front(1)) { + M->moveBefore(Where); + Where = M; + } + } + + return Move.Main.size() + Move.Deps.size() > 1; +} + +auto AlignVectors::realignGroup(const MoveGroup &Move) const -> bool { + // TODO: Needs support for masked loads/stores of "scalar" vectors. + if (!Move.IsHvx) + return false; + + // Return the element with the maximum alignment from Range, + // where GetValue obtains the value to compare from an element. + auto getMaxOf = [](auto Range, auto GetValue) { + return *std::max_element( + Range.begin(), Range.end(), + [&GetValue](auto &A, auto &B) { return GetValue(A) < GetValue(B); }); + }; + + const AddrList &BaseInfos = AddrGroups.at(Move.Base); + + // Conceptually, there is a vector of N bytes covering the addresses + // starting from the minimum offset (i.e. Base.Addr+Start). This vector + // represents a contiguous memory region that spans all accessed memory + // locations. + // The correspondence between loaded or stored values will be expressed + // in terms of this vector. For example, the 0th element of the vector + // from the Base address info will start at byte Start from the beginning + // of this conceptual vector. + // + // This vector will be loaded/stored starting at the nearest down-aligned + // address and the amount od the down-alignment will be AlignVal: + // valign(load_vector(align_down(Base+Start)), AlignVal) + + std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end()); + AddrList MoveInfos; + llvm::copy_if( + BaseInfos, std::back_inserter(MoveInfos), + [&TestSet](const AddrInfo &AI) { return TestSet.count(AI.Inst); }); + + // Maximum alignment present in the whole address group. + const AddrInfo &WithMaxAlign = + getMaxOf(BaseInfos, [](const AddrInfo &AI) { return AI.HaveAlign; }); + Align MaxGiven = WithMaxAlign.HaveAlign; + + // Minimum alignment present in the move address group. + const AddrInfo &WithMinOffset = + getMaxOf(MoveInfos, [](const AddrInfo &AI) { return -AI.Offset; }); + + const AddrInfo &WithMaxNeeded = + getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.NeedAlign; }); + Align MinNeeded = WithMaxNeeded.NeedAlign; + + // Set the builder at the top instruction in the move group. + Instruction *TopIn = Move.IsLoad ? Move.Main.front() : Move.Main.back(); + IRBuilder<> Builder(TopIn); + Value *AlignAddr = nullptr; // Actual aligned address. + Value *AlignVal = nullptr; // Right-shift amount (for valign). + + if (MinNeeded <= MaxGiven) { + int Start = WithMinOffset.Offset; + int OffAtMax = WithMaxAlign.Offset; + // Shift the offset of the maximally aligned instruction (OffAtMax) + // back by just enough multiples of the required alignment to cover the + // distance from Start to OffAtMax. + // Calculate the address adjustment amount based on the address with the + // maximum alignment. This is to allow a simple gep instruction instead + // of potential bitcasts to i8*. + int Adjust = -alignTo(OffAtMax - Start, MinNeeded.value()); + AlignAddr = createAdjustedPointer(Builder, WithMaxAlign.Addr, + WithMaxAlign.ValTy, Adjust); + int Diff = Start - (OffAtMax + Adjust); + AlignVal = HVC.getConstInt(Diff); + // Sanity. + assert(Diff >= 0); + assert(static_cast<decltype(MinNeeded.value())>(Diff) < MinNeeded.value()); + } else { + // WithMinOffset is the lowest address in the group, + // WithMinOffset.Addr = Base+Start. + // Align instructions for both HVX (V6_valign) and scalar (S2_valignrb) + // mask off unnecessary bits, so it's ok to just the original pointer as + // the alignment amount. + // Do an explicit down-alignment of the address to avoid creating an + // aligned instruction with an address that is not really aligned. + AlignAddr = createAlignedPointer(Builder, WithMinOffset.Addr, + WithMinOffset.ValTy, MinNeeded.value()); + AlignVal = Builder.CreatePtrToInt(WithMinOffset.Addr, HVC.getIntTy()); + } + + ByteSpan VSpan; + for (const AddrInfo &AI : MoveInfos) { + VSpan.Blocks.emplace_back(AI.Inst, HVC.getSizeOf(AI.ValTy), + AI.Offset - WithMinOffset.Offset); + } + + // The aligned loads/stores will use blocks that are either scalars, + // or HVX vectors. Let "sector" be the unified term for such a block. + // blend(scalar, vector) -> sector... + int ScLen = Move.IsHvx ? HVC.HST.getVectorLength() + : std::max<int>(MinNeeded.value(), 4); + assert(!Move.IsHvx || ScLen == 64 || ScLen == 128); + assert(Move.IsHvx || ScLen == 4 || ScLen == 8); + + Type *SecTy = HVC.getByteTy(ScLen); + int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen; + + if (Move.IsLoad) { + ByteSpan ASpan; + auto *True = HVC.getFullValue(HVC.getBoolTy(ScLen)); + auto *Undef = UndefValue::get(SecTy); + + for (int i = 0; i != NumSectors + 1; ++i) { + Value *Ptr = createAdjustedPointer(Builder, AlignAddr, SecTy, i * ScLen); + // FIXME: generate a predicated load? + Value *Load = createAlignedLoad(Builder, SecTy, Ptr, ScLen, True, Undef); + ASpan.Blocks.emplace_back(Load, ScLen, i * ScLen); + } + + for (int j = 0; j != NumSectors; ++j) { + ASpan[j].Seg.Val = HVC.vralignb(Builder, ASpan[j].Seg.Val, + ASpan[j + 1].Seg.Val, AlignVal); + } + + for (ByteSpan::Block &B : VSpan) { + ByteSpan Section = ASpan.section(B.Pos, B.Seg.Size).shift(-B.Pos); + Value *Accum = UndefValue::get(HVC.getByteTy(B.Seg.Size)); + for (ByteSpan::Block &S : Section) { + Value *Pay = HVC.vbytes(Builder, getPayload(S.Seg.Val)); + Accum = + HVC.insertb(Builder, Accum, Pay, S.Seg.Start, S.Seg.Size, S.Pos); + } + // Instead of casting everything to bytes for the vselect, cast to the + // original value type. This will avoid complications with casting masks. + // For example, in cases when the original mask applied to i32, it could + // be converted to a mask applicable to i8 via pred_typecast intrinsic, + // but if the mask is not exactly of HVX length, extra handling would be + // needed to make it work. + Type *ValTy = getPayload(B.Seg.Val)->getType(); + Value *Cast = Builder.CreateBitCast(Accum, ValTy); + Value *Sel = Builder.CreateSelect(getMask(B.Seg.Val), Cast, + getPassThrough(B.Seg.Val)); + B.Seg.Val->replaceAllUsesWith(Sel); + } + } else { + // Stores. + ByteSpan ASpanV, ASpanM; + + // Return a vector value corresponding to the input value Val: + // either <1 x Val> for scalar Val, or Val itself for vector Val. + auto MakeVec = [](IRBuilder<> &Builder, Value *Val) -> Value * { + Type *Ty = Val->getType(); + if (Ty->isVectorTy()) + return Val; + auto *VecTy = VectorType::get(Ty, 1, /*Scalable*/ false); + return Builder.CreateBitCast(Val, VecTy); + }; + + // Create an extra "undef" sector at the beginning and at the end. + // They will be used as the left/right filler in the vlalign step. + for (int i = -1; i != NumSectors + 1; ++i) { + // For stores, the size of each section is an aligned vector length. + // Adjust the store offsets relative to the section start offset. + ByteSpan Section = VSpan.section(i * ScLen, ScLen).shift(-i * ScLen); + Value *AccumV = UndefValue::get(SecTy); + Value *AccumM = HVC.getNullValue(SecTy); + for (ByteSpan::Block &S : Section) { + Value *Pay = getPayload(S.Seg.Val); + Value *Mask = HVC.rescale(Builder, MakeVec(Builder, getMask(S.Seg.Val)), + Pay->getType(), HVC.getByteTy()); + AccumM = HVC.insertb(Builder, AccumM, HVC.vbytes(Builder, Mask), + S.Seg.Start, S.Seg.Size, S.Pos); + AccumV = HVC.insertb(Builder, AccumV, HVC.vbytes(Builder, Pay), + S.Seg.Start, S.Seg.Size, S.Pos); + } + ASpanV.Blocks.emplace_back(AccumV, ScLen, i * ScLen); + ASpanM.Blocks.emplace_back(AccumM, ScLen, i * ScLen); + } + + // vlalign + for (int j = 1; j != NumSectors + 2; ++j) { + ASpanV[j - 1].Seg.Val = HVC.vlalignb(Builder, ASpanV[j - 1].Seg.Val, + ASpanV[j].Seg.Val, AlignVal); + ASpanM[j - 1].Seg.Val = HVC.vlalignb(Builder, ASpanM[j - 1].Seg.Val, + ASpanM[j].Seg.Val, AlignVal); + } + + for (int i = 0; i != NumSectors + 1; ++i) { + Value *Ptr = createAdjustedPointer(Builder, AlignAddr, SecTy, i * ScLen); + Value *Val = ASpanV[i].Seg.Val; + Value *Mask = ASpanM[i].Seg.Val; // bytes + if (!HVC.isUndef(Val) && !HVC.isZero(Mask)) + createAlignedStore(Builder, Val, Ptr, ScLen, HVC.vlsb(Builder, Mask)); + } + } + + for (auto *Inst : Move.Main) + Inst->eraseFromParent(); + + return true; +} + +auto AlignVectors::run() -> bool { + if (!createAddressGroups()) + return false; + + bool Changed = false; + MoveList LoadGroups, StoreGroups; + + for (auto &G : AddrGroups) { + llvm::append_range(LoadGroups, createLoadGroups(G.second)); + llvm::append_range(StoreGroups, createStoreGroups(G.second)); + } + + for (auto &M : LoadGroups) + Changed |= move(M); + for (auto &M : StoreGroups) + Changed |= move(M); + + for (auto &M : LoadGroups) + Changed |= realignGroup(M); + for (auto &M : StoreGroups) + Changed |= realignGroup(M); + + return Changed; +} + +// --- End AlignVectors + +auto HexagonVectorCombine::run() -> bool { + if (!HST.useHVXOps()) + return false; + + bool Changed = AlignVectors(*this).run(); + return Changed; +} + +auto HexagonVectorCombine::getIntTy() const -> IntegerType * { + return Type::getInt32Ty(F.getContext()); +} + +auto HexagonVectorCombine::getByteTy(int ElemCount) const -> Type * { + assert(ElemCount >= 0); + IntegerType *ByteTy = Type::getInt8Ty(F.getContext()); + if (ElemCount == 0) + return ByteTy; + return VectorType::get(ByteTy, ElemCount, /*Scalable*/ false); +} + +auto HexagonVectorCombine::getBoolTy(int ElemCount) const -> Type * { + assert(ElemCount >= 0); + IntegerType *BoolTy = Type::getInt1Ty(F.getContext()); + if (ElemCount == 0) + return BoolTy; + return VectorType::get(BoolTy, ElemCount, /*Scalable*/ false); +} + +auto HexagonVectorCombine::getConstInt(int Val) const -> ConstantInt * { + return ConstantInt::getSigned(getIntTy(), Val); +} + +auto HexagonVectorCombine::isZero(const Value *Val) const -> bool { + if (auto *C = dyn_cast<Constant>(Val)) + return C->isZeroValue(); + return false; +} + +auto HexagonVectorCombine::getIntValue(const Value *Val) const + -> Optional<APInt> { + if (auto *CI = dyn_cast<ConstantInt>(Val)) + return CI->getValue(); + return None; +} + +auto HexagonVectorCombine::isUndef(const Value *Val) const -> bool { + return isa<UndefValue>(Val); +} + +auto HexagonVectorCombine::getSizeOf(const Value *Val) const -> int { + return getSizeOf(Val->getType()); +} + +auto HexagonVectorCombine::getSizeOf(const Type *Ty) const -> int { + return DL.getTypeStoreSize(const_cast<Type *>(Ty)).getFixedValue(); +} + +auto HexagonVectorCombine::getTypeAlignment(Type *Ty) const -> int { + // The actual type may be shorter than the HVX vector, so determine + // the alignment based on subtarget info. + if (HST.isTypeForHVX(Ty)) + return HST.getVectorLength(); + return DL.getABITypeAlign(Ty).value(); +} + +auto HexagonVectorCombine::getNullValue(Type *Ty) const -> Constant * { + assert(Ty->isIntOrIntVectorTy()); + auto Zero = ConstantInt::get(Ty->getScalarType(), 0); + if (auto *VecTy = dyn_cast<VectorType>(Ty)) + return ConstantVector::getSplat(VecTy->getElementCount(), Zero); + return Zero; +} + +auto HexagonVectorCombine::getFullValue(Type *Ty) const -> Constant * { + assert(Ty->isIntOrIntVectorTy()); + auto Minus1 = ConstantInt::get(Ty->getScalarType(), -1); + if (auto *VecTy = dyn_cast<VectorType>(Ty)) + return ConstantVector::getSplat(VecTy->getElementCount(), Minus1); + return Minus1; +} + +// Insert bytes [Start..Start+Length) of Src into Dst at byte Where. +auto HexagonVectorCombine::insertb(IRBuilder<> &Builder, Value *Dst, Value *Src, + int Start, int Length, int Where) const + -> Value * { + assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType())); + int SrcLen = getSizeOf(Src); + int DstLen = getSizeOf(Dst); + assert(0 <= Start && Start + Length <= SrcLen); + assert(0 <= Where && Where + Length <= DstLen); + + int P2Len = PowerOf2Ceil(SrcLen | DstLen); + auto *Undef = UndefValue::get(getByteTy()); + Value *P2Src = vresize(Builder, Src, P2Len, Undef); + Value *P2Dst = vresize(Builder, Dst, P2Len, Undef); + + SmallVector<int, 256> SMask(P2Len); + for (int i = 0; i != P2Len; ++i) { + // If i is in [Where, Where+Length), pick Src[Start+(i-Where)]. + // Otherwise, pick Dst[i]; + SMask[i] = + (Where <= i && i < Where + Length) ? P2Len + Start + (i - Where) : i; + } + + Value *P2Insert = Builder.CreateShuffleVector(P2Dst, P2Src, SMask); + return vresize(Builder, P2Insert, DstLen, Undef); +} + +auto HexagonVectorCombine::vlalignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, + Value *Amt) const -> Value * { + assert(Lo->getType() == Hi->getType() && "Argument type mismatch"); + assert(isSectorTy(Hi->getType())); + if (isZero(Amt)) + return Hi; + int VecLen = getSizeOf(Hi); + if (auto IntAmt = getIntValue(Amt)) + return getElementRange(Builder, Lo, Hi, VecLen - IntAmt->getSExtValue(), + VecLen); + + if (HST.isTypeForHVX(Hi->getType())) { + int HwLen = HST.getVectorLength(); + assert(VecLen == HwLen && "Expecting an exact HVX type"); + Intrinsic::ID V6_vlalignb = HwLen == 64 + ? Intrinsic::hexagon_V6_vlalignb + : Intrinsic::hexagon_V6_vlalignb_128B; + return createHvxIntrinsic(Builder, V6_vlalignb, Hi->getType(), + {Hi, Lo, Amt}); + } + + if (VecLen == 4) { + Value *Pair = concat(Builder, {Lo, Hi}); + Value *Shift = Builder.CreateLShr(Builder.CreateShl(Pair, Amt), 32); + Value *Trunc = Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext())); + return Builder.CreateBitCast(Trunc, Hi->getType()); + } + if (VecLen == 8) { + Value *Sub = Builder.CreateSub(getConstInt(VecLen), Amt); + return vralignb(Builder, Lo, Hi, Sub); + } + llvm_unreachable("Unexpected vector length"); +} + +auto HexagonVectorCombine::vralignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, + Value *Amt) const -> Value * { + assert(Lo->getType() == Hi->getType() && "Argument type mismatch"); + assert(isSectorTy(Lo->getType())); + if (isZero(Amt)) + return Lo; + int VecLen = getSizeOf(Lo); + if (auto IntAmt = getIntValue(Amt)) + return getElementRange(Builder, Lo, Hi, IntAmt->getSExtValue(), VecLen); + + if (HST.isTypeForHVX(Lo->getType())) { + int HwLen = HST.getVectorLength(); + assert(VecLen == HwLen && "Expecting an exact HVX type"); + Intrinsic::ID V6_valignb = HwLen == 64 ? Intrinsic::hexagon_V6_valignb + : Intrinsic::hexagon_V6_valignb_128B; + return createHvxIntrinsic(Builder, V6_valignb, Lo->getType(), + {Hi, Lo, Amt}); + } + + if (VecLen == 4) { + Value *Pair = concat(Builder, {Lo, Hi}); + Value *Shift = Builder.CreateLShr(Pair, Amt); + Value *Trunc = Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext())); + return Builder.CreateBitCast(Trunc, Lo->getType()); + } + if (VecLen == 8) { + Type *Int64Ty = Type::getInt64Ty(F.getContext()); + Value *Lo64 = Builder.CreateBitCast(Lo, Int64Ty); + Value *Hi64 = Builder.CreateBitCast(Hi, Int64Ty); + Function *FI = Intrinsic::getDeclaration(F.getParent(), + Intrinsic::hexagon_S2_valignrb); + Value *Call = Builder.CreateCall(FI, {Hi64, Lo64, Amt}); + return Builder.CreateBitCast(Call, Lo->getType()); + } + llvm_unreachable("Unexpected vector length"); +} + +// Concatenates a sequence of vectors of the same type. +auto HexagonVectorCombine::concat(IRBuilder<> &Builder, + ArrayRef<Value *> Vecs) const -> Value * { + assert(!Vecs.empty()); + SmallVector<int, 256> SMask; + std::vector<Value *> Work[2]; + int ThisW = 0, OtherW = 1; + + Work[ThisW].assign(Vecs.begin(), Vecs.end()); + while (Work[ThisW].size() > 1) { + auto *Ty = cast<VectorType>(Work[ThisW].front()->getType()); + int ElemCount = Ty->getElementCount().getFixedValue(); + SMask.resize(ElemCount * 2); + std::iota(SMask.begin(), SMask.end(), 0); + + Work[OtherW].clear(); + if (Work[ThisW].size() % 2 != 0) + Work[ThisW].push_back(UndefValue::get(Ty)); + for (int i = 0, e = Work[ThisW].size(); i < e; i += 2) { + Value *Joined = Builder.CreateShuffleVector(Work[ThisW][i], + Work[ThisW][i + 1], SMask); + Work[OtherW].push_back(Joined); + } + std::swap(ThisW, OtherW); + } + + // Since there may have been some undefs appended to make shuffle operands + // have the same type, perform the last shuffle to only pick the original + // elements. + SMask.resize(Vecs.size() * getSizeOf(Vecs.front()->getType())); + std::iota(SMask.begin(), SMask.end(), 0); + Value *Total = Work[OtherW].front(); + return Builder.CreateShuffleVector(Total, SMask); +} + +auto HexagonVectorCombine::vresize(IRBuilder<> &Builder, Value *Val, + int NewSize, Value *Pad) const -> Value * { + assert(isa<VectorType>(Val->getType())); + auto *ValTy = cast<VectorType>(Val->getType()); + assert(ValTy->getElementType() == Pad->getType()); + + int CurSize = ValTy->getElementCount().getFixedValue(); + if (CurSize == NewSize) + return Val; + // Truncate? + if (CurSize > NewSize) + return getElementRange(Builder, Val, /*Unused*/ Val, 0, NewSize); + // Extend. + SmallVector<int, 128> SMask(NewSize); + std::iota(SMask.begin(), SMask.begin() + CurSize, 0); + std::fill(SMask.begin() + CurSize, SMask.end(), CurSize); + Value *PadVec = Builder.CreateVectorSplat(CurSize, Pad); + return Builder.CreateShuffleVector(Val, PadVec, SMask); +} + +auto HexagonVectorCombine::rescale(IRBuilder<> &Builder, Value *Mask, + Type *FromTy, Type *ToTy) const -> Value * { + // Mask is a vector <N x i1>, where each element corresponds to an + // element of FromTy. Remap it so that each element will correspond + // to an element of ToTy. + assert(isa<VectorType>(Mask->getType())); + + Type *FromSTy = FromTy->getScalarType(); + Type *ToSTy = ToTy->getScalarType(); + if (FromSTy == ToSTy) + return Mask; + + int FromSize = getSizeOf(FromSTy); + int ToSize = getSizeOf(ToSTy); + assert(FromSize % ToSize == 0 || ToSize % FromSize == 0); + + auto *MaskTy = cast<VectorType>(Mask->getType()); + int FromCount = MaskTy->getElementCount().getFixedValue(); + int ToCount = (FromCount * FromSize) / ToSize; + assert((FromCount * FromSize) % ToSize == 0); + + // Mask <N x i1> -> sext to <N x FromTy> -> bitcast to <M x ToTy> -> + // -> trunc to <M x i1>. + Value *Ext = Builder.CreateSExt( + Mask, VectorType::get(FromSTy, FromCount, /*Scalable*/ false)); + Value *Cast = Builder.CreateBitCast( + Ext, VectorType::get(ToSTy, ToCount, /*Scalable*/ false)); + return Builder.CreateTrunc( + Cast, VectorType::get(getBoolTy(), ToCount, /*Scalable*/ false)); +} + +// Bitcast to bytes, and return least significant bits. +auto HexagonVectorCombine::vlsb(IRBuilder<> &Builder, Value *Val) const + -> Value * { + Type *ScalarTy = Val->getType()->getScalarType(); + if (ScalarTy == getBoolTy()) + return Val; + + Value *Bytes = vbytes(Builder, Val); + if (auto *VecTy = dyn_cast<VectorType>(Bytes->getType())) + return Builder.CreateTrunc(Bytes, getBoolTy(getSizeOf(VecTy))); + // If Bytes is a scalar (i.e. Val was a scalar byte), return i1, not + // <1 x i1>. + return Builder.CreateTrunc(Bytes, getBoolTy()); +} + +// Bitcast to bytes for non-bool. For bool, convert i1 -> i8. +auto HexagonVectorCombine::vbytes(IRBuilder<> &Builder, Value *Val) const + -> Value * { + Type *ScalarTy = Val->getType()->getScalarType(); + if (ScalarTy == getByteTy()) + return Val; + + if (ScalarTy != getBoolTy()) + return Builder.CreateBitCast(Val, getByteTy(getSizeOf(Val))); + // For bool, return a sext from i1 to i8. + if (auto *VecTy = dyn_cast<VectorType>(Val->getType())) + return Builder.CreateSExt(Val, VectorType::get(getByteTy(), VecTy)); + return Builder.CreateSExt(Val, getByteTy()); +} + +auto HexagonVectorCombine::createHvxIntrinsic(IRBuilder<> &Builder, + Intrinsic::ID IntID, Type *RetTy, + ArrayRef<Value *> Args) const + -> Value * { + int HwLen = HST.getVectorLength(); + Type *BoolTy = Type::getInt1Ty(F.getContext()); + Type *Int32Ty = Type::getInt32Ty(F.getContext()); + // HVX vector -> v16i32/v32i32 + // HVX vector predicate -> v512i1/v1024i1 + auto getTypeForIntrin = [&](Type *Ty) -> Type * { + if (HST.isTypeForHVX(Ty, /*IncludeBool*/ true)) { + Type *ElemTy = cast<VectorType>(Ty)->getElementType(); + if (ElemTy == Int32Ty) + return Ty; + if (ElemTy == BoolTy) + return VectorType::get(BoolTy, 8 * HwLen, /*Scalable*/ false); + return VectorType::get(Int32Ty, HwLen / 4, /*Scalable*/ false); + } + // Non-HVX type. It should be a scalar. + assert(Ty == Int32Ty || Ty->isIntegerTy(64)); + return Ty; + }; + + auto getCast = [&](IRBuilder<> &Builder, Value *Val, + Type *DestTy) -> Value * { + Type *SrcTy = Val->getType(); + if (SrcTy == DestTy) + return Val; + if (HST.isTypeForHVX(SrcTy, /*IncludeBool*/ true)) { + if (cast<VectorType>(SrcTy)->getElementType() == BoolTy) { + // This should take care of casts the other way too, for example + // v1024i1 -> v32i1. + Intrinsic::ID TC = HwLen == 64 + ? Intrinsic::hexagon_V6_pred_typecast + : Intrinsic::hexagon_V6_pred_typecast_128B; + Function *FI = Intrinsic::getDeclaration(F.getParent(), TC, + {DestTy, Val->getType()}); + return Builder.CreateCall(FI, {Val}); + } + // Non-predicate HVX vector. + return Builder.CreateBitCast(Val, DestTy); + } + // Non-HVX type. It should be a scalar, and it should already have + // a valid type. + llvm_unreachable("Unexpected type"); + }; + + SmallVector<Value *, 4> IntOps; + for (Value *A : Args) + IntOps.push_back(getCast(Builder, A, getTypeForIntrin(A->getType()))); + Function *FI = Intrinsic::getDeclaration(F.getParent(), IntID); + Value *Call = Builder.CreateCall(FI, IntOps); + + Type *CallTy = Call->getType(); + if (CallTy == RetTy) + return Call; + // Scalar types should have RetTy matching the call return type. + assert(HST.isTypeForHVX(CallTy, /*IncludeBool*/ true)); + if (cast<VectorType>(CallTy)->getElementType() == BoolTy) + return getCast(Builder, Call, RetTy); + return Builder.CreateBitCast(Call, RetTy); +} + +auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0, + Value *Ptr1) const + -> Optional<int> { + struct Builder : IRBuilder<> { + Builder(BasicBlock *B) : IRBuilder<>(B) {} + ~Builder() { + for (Instruction *I : llvm::reverse(ToErase)) + I->eraseFromParent(); + } + SmallVector<Instruction *, 8> ToErase; + }; + +#define CallBuilder(B, F) \ + [&](auto &B_) { \ + Value *V = B_.F; \ + if (auto *I = dyn_cast<Instruction>(V)) \ + B_.ToErase.push_back(I); \ + return V; \ + }(B) + + auto Simplify = [&](Value *V) { + if (auto *I = dyn_cast<Instruction>(V)) { + SimplifyQuery Q(DL, &TLI, &DT, &AC, I); + if (Value *S = SimplifyInstruction(I, Q)) + return S; + } + return V; + }; + + auto StripBitCast = [](Value *V) { + while (auto *C = dyn_cast<BitCastInst>(V)) + V = C->getOperand(0); + return V; + }; + + Ptr0 = StripBitCast(Ptr0); + Ptr1 = StripBitCast(Ptr1); + if (!isa<GetElementPtrInst>(Ptr0) || !isa<GetElementPtrInst>(Ptr1)) + return None; + + auto *Gep0 = cast<GetElementPtrInst>(Ptr0); + auto *Gep1 = cast<GetElementPtrInst>(Ptr1); + if (Gep0->getPointerOperand() != Gep1->getPointerOperand()) + return None; + + Builder B(Gep0->getParent()); + Value *BasePtr = Gep0->getPointerOperand(); + int Scale = DL.getTypeStoreSize(BasePtr->getType()->getPointerElementType()); + + // FIXME: for now only check GEPs with a single index. + if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2) + return None; + + Value *Idx0 = Gep0->getOperand(1); + Value *Idx1 = Gep1->getOperand(1); + + // First, try to simplify the subtraction directly. + if (auto *Diff = dyn_cast<ConstantInt>( + Simplify(CallBuilder(B, CreateSub(Idx0, Idx1))))) + return Diff->getSExtValue() * Scale; + + KnownBits Known0 = computeKnownBits(Idx0, DL, 0, &AC, Gep0, &DT); + KnownBits Known1 = computeKnownBits(Idx1, DL, 0, &AC, Gep1, &DT); + APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One); + if (Unknown.isAllOnesValue()) + return None; + + Value *MaskU = ConstantInt::get(Idx0->getType(), Unknown); + Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU))); + Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU))); + Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1))); + int Diff0 = 0; + if (auto *C = dyn_cast<ConstantInt>(SubU)) { + Diff0 = C->getSExtValue(); + } else { + return None; + } + + Value *MaskK = ConstantInt::get(MaskU->getType(), ~Unknown); + Value *AndK0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskK))); + Value *AndK1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskK))); + Value *SubK = Simplify(CallBuilder(B, CreateSub(AndK0, AndK1))); + int Diff1 = 0; + if (auto *C = dyn_cast<ConstantInt>(SubK)) { + Diff1 = C->getSExtValue(); + } else { + return None; + } + + return (Diff0 + Diff1) * Scale; + +#undef CallBuilder +} + +template <typename T> +auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In, + BasicBlock::const_iterator To, + const T &Ignore) const + -> bool { + auto getLocOrNone = [this](const Instruction &I) -> Optional<MemoryLocation> { + if (const auto *II = dyn_cast<IntrinsicInst>(&I)) { + switch (II->getIntrinsicID()) { + case Intrinsic::masked_load: + return MemoryLocation::getForArgument(II, 0, TLI); + case Intrinsic::masked_store: + return MemoryLocation::getForArgument(II, 1, TLI); + } + } + return MemoryLocation::getOrNone(&I); + }; + + // The source and the destination must be in the same basic block. + const BasicBlock &Block = *In.getParent(); + assert(Block.begin() == To || Block.end() == To || To->getParent() == &Block); + // No PHIs. + if (isa<PHINode>(In) || (To != Block.end() && isa<PHINode>(*To))) + return false; + + if (!mayBeMemoryDependent(In)) + return true; + bool MayWrite = In.mayWriteToMemory(); + auto MaybeLoc = getLocOrNone(In); + + auto From = In.getIterator(); + if (From == To) + return true; + bool MoveUp = (To != Block.end() && To->comesBefore(&In)); + auto Range = + MoveUp ? std::make_pair(To, From) : std::make_pair(std::next(From), To); + for (auto It = Range.first; It != Range.second; ++It) { + const Instruction &I = *It; + if (llvm::is_contained(Ignore, &I)) + continue; + // Parts based on isSafeToMoveBefore from CoveMoverUtils.cpp. + if (I.mayThrow()) + return false; + if (auto *CB = dyn_cast<CallBase>(&I)) { + if (!CB->hasFnAttr(Attribute::WillReturn)) + return false; + if (!CB->hasFnAttr(Attribute::NoSync)) + return false; + } + if (I.mayReadOrWriteMemory()) { + auto MaybeLocI = getLocOrNone(I); + if (MayWrite || I.mayWriteToMemory()) { + if (!MaybeLoc || !MaybeLocI) + return false; + if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI)) + return false; + } + } + } + return true; +} + +#ifndef NDEBUG +auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool { + if (auto *VecTy = dyn_cast<VectorType>(Ty)) + return VecTy->getElementType() == getByteTy(); + return false; +} + +auto HexagonVectorCombine::isSectorTy(Type *Ty) const -> bool { + if (!isByteVecTy(Ty)) + return false; + int Size = getSizeOf(Ty); + if (HST.isTypeForHVX(Ty)) + return Size == static_cast<int>(HST.getVectorLength()); + return Size == 4 || Size == 8; +} +#endif + +auto HexagonVectorCombine::getElementRange(IRBuilder<> &Builder, Value *Lo, + Value *Hi, int Start, + int Length) const -> Value * { + assert(0 <= Start && Start < Length); + SmallVector<int, 128> SMask(Length); + std::iota(SMask.begin(), SMask.end(), Start); + return Builder.CreateShuffleVector(Lo, Hi, SMask); +} + +// Pass management. + +namespace llvm { +void initializeHexagonVectorCombineLegacyPass(PassRegistry &); +FunctionPass *createHexagonVectorCombineLegacyPass(); +} // namespace llvm + +namespace { +class HexagonVectorCombineLegacy : public FunctionPass { +public: + static char ID; + + HexagonVectorCombineLegacy() : FunctionPass(ID) {} + + StringRef getPassName() const override { return "Hexagon Vector Combine"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired<AAResultsWrapperPass>(); + AU.addRequired<AssumptionCacheTracker>(); + AU.addRequired<DominatorTreeWrapperPass>(); + AU.addRequired<TargetLibraryInfoWrapperPass>(); + AU.addRequired<TargetPassConfig>(); + FunctionPass::getAnalysisUsage(AU); + } + + bool runOnFunction(Function &F) override { + AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults(); + AssumptionCache &AC = + getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); + DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); + TargetLibraryInfo &TLI = + getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); + auto &TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>(); + HexagonVectorCombine HVC(F, AA, AC, DT, TLI, TM); + return HVC.run(); + } +}; +} // namespace + +char HexagonVectorCombineLegacy::ID = 0; + +INITIALIZE_PASS_BEGIN(HexagonVectorCombineLegacy, DEBUG_TYPE, + "Hexagon Vector Combine", false, false) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) +INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_END(HexagonVectorCombineLegacy, DEBUG_TYPE, + "Hexagon Vector Combine", false, false) + +FunctionPass *llvm::createHexagonVectorCombineLegacyPass() { + return new HexagonVectorCombineLegacy(); +} diff --git a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp index 42451e02ba36..310536458de9 100644 --- a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp +++ b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp @@ -11,110 +11,9 @@ // to identify loop carried dependences. This is scalar replacement for vector // types. // -//----------------------------------------------------------------------------- -// Motivation: Consider the case where we have the following loop structure. -// -// Loop: -// t0 = a[i]; -// t1 = f(t0); -// t2 = g(t1); -// ... -// t3 = a[i+1]; -// t4 = f(t3); -// t5 = g(t4); -// t6 = op(t2, t5) -// cond_branch <Loop> -// -// This can be converted to -// t00 = a[0]; -// t10 = f(t00); -// t20 = g(t10); -// Loop: -// t2 = t20; -// t3 = a[i+1]; -// t4 = f(t3); -// t5 = g(t4); -// t6 = op(t2, t5) -// t20 = t5 -// cond_branch <Loop> -// -// SROA does a good job of reusing a[i+1] as a[i] in the next iteration. -// Such a loop comes to this pass in the following form. -// -// LoopPreheader: -// X0 = a[0]; -// Loop: -// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> -// t1 = f(X2) <-- I1 -// t2 = g(t1) -// ... -// X1 = a[i+1] -// t4 = f(X1) <-- I2 -// t5 = g(t4) -// t6 = op(t2, t5) -// cond_branch <Loop> -// -// In this pass, we look for PHIs such as X2 whose incoming values come only -// from the Loop Preheader and over the backedge and additionaly, both these -// values are the results of the same operation in terms of opcode. We call such -// a PHI node a dependence chain or DepChain. In this case, the dependence of X2 -// over X1 is carried over only one iteration and so the DepChain is only one -// PHI node long. -// -// Then, we traverse the uses of the PHI (X2) and the uses of the value of the -// PHI coming over the backedge (X1). We stop at the first pair of such users -// I1 (of X2) and I2 (of X1) that meet the following conditions. -// 1. I1 and I2 are the same operation, but with different operands. -// 2. X2 and X1 are used at the same operand number in the two instructions. -// 3. All other operands Op1 of I1 and Op2 of I2 are also such that there is a -// a DepChain from Op1 to Op2 of the same length as that between X2 and X1. -// -// We then make the following transformation -// LoopPreheader: -// X0 = a[0]; -// Y0 = f(X0); -// Loop: -// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> -// Y2 = PHI<(Y0, LoopPreheader), (t4, Loop)> -// t1 = f(X2) <-- Will be removed by DCE. -// t2 = g(Y2) -// ... -// X1 = a[i+1] -// t4 = f(X1) -// t5 = g(t4) -// t6 = op(t2, t5) -// cond_branch <Loop> -// -// We proceed until we cannot find any more such instructions I1 and I2. -// -// --- DepChains & Loop carried dependences --- -// Consider a single basic block loop such as -// -// LoopPreheader: -// X0 = ... -// Y0 = ... -// Loop: -// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> -// Y2 = PHI<(Y0, LoopPreheader), (X2, Loop)> -// ... -// X1 = ... -// ... -// cond_branch <Loop> -// -// Then there is a dependence between X2 and X1 that goes back one iteration, -// i.e. X1 is used as X2 in the very next iteration. We represent this as a -// DepChain from X2 to X1 (X2->X1). -// Similarly, there is a dependence between Y2 and X1 that goes back two -// iterations. X1 is used as Y2 two iterations after it is computed. This is -// represented by a DepChain as (Y2->X2->X1). -// -// A DepChain has the following properties. -// 1. Num of edges in DepChain = Number of Instructions in DepChain = Number of -// iterations of carried dependence + 1. -// 2. All instructions in the DepChain except the last are PHIs. -// //===----------------------------------------------------------------------===// +#include "HexagonVectorLoopCarriedReuse.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -161,8 +60,8 @@ static cl::opt<int> HexagonVLCRIterationLim("hexagon-vlcr-iteration-lim", namespace llvm { -void initializeHexagonVectorLoopCarriedReusePass(PassRegistry&); -Pass *createHexagonVectorLoopCarriedReusePass(); +void initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PassRegistry &); +Pass *createHexagonVectorLoopCarriedReuseLegacyPass(); } // end namespace llvm @@ -262,13 +161,13 @@ namespace { return OS; } - class HexagonVectorLoopCarriedReuse : public LoopPass { + class HexagonVectorLoopCarriedReuseLegacyPass : public LoopPass { public: static char ID; - explicit HexagonVectorLoopCarriedReuse() : LoopPass(ID) { + explicit HexagonVectorLoopCarriedReuseLegacyPass() : LoopPass(ID) { PassRegistry *PR = PassRegistry::getPassRegistry(); - initializeHexagonVectorLoopCarriedReusePass(*PR); + initializeHexagonVectorLoopCarriedReuseLegacyPassPass(*PR); } StringRef getPassName() const override { @@ -276,7 +175,6 @@ namespace { } void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired<LoopInfoWrapperPass>(); AU.addRequiredID(LoopSimplifyID); AU.addRequiredID(LCSSAID); AU.addPreservedID(LCSSAID); @@ -284,6 +182,13 @@ namespace { } bool runOnLoop(Loop *L, LPPassManager &LPM) override; + }; + + class HexagonVectorLoopCarriedReuse { + public: + HexagonVectorLoopCarriedReuse(Loop *L) : CurLoop(L){}; + + bool run(); private: SetVector<DepChain *> Dependences; @@ -305,33 +210,49 @@ namespace { } // end anonymous namespace -char HexagonVectorLoopCarriedReuse::ID = 0; +char HexagonVectorLoopCarriedReuseLegacyPass::ID = 0; -INITIALIZE_PASS_BEGIN(HexagonVectorLoopCarriedReuse, "hexagon-vlcr", - "Hexagon-specific predictive commoning for HVX vectors", false, false) -INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_BEGIN(HexagonVectorLoopCarriedReuseLegacyPass, "hexagon-vlcr", + "Hexagon-specific predictive commoning for HVX vectors", + false, false) INITIALIZE_PASS_DEPENDENCY(LoopSimplify) INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass) -INITIALIZE_PASS_END(HexagonVectorLoopCarriedReuse, "hexagon-vlcr", - "Hexagon-specific predictive commoning for HVX vectors", false, false) +INITIALIZE_PASS_END(HexagonVectorLoopCarriedReuseLegacyPass, "hexagon-vlcr", + "Hexagon-specific predictive commoning for HVX vectors", + false, false) + +PreservedAnalyses +HexagonVectorLoopCarriedReusePass::run(Loop &L, LoopAnalysisManager &LAM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U) { + HexagonVectorLoopCarriedReuse Vlcr(&L); + if (!Vlcr.run()) + return PreservedAnalyses::all(); + PreservedAnalyses PA; + PA.preserveSet<CFGAnalyses>(); + return PA; +} -bool HexagonVectorLoopCarriedReuse::runOnLoop(Loop *L, LPPassManager &LPM) { +bool HexagonVectorLoopCarriedReuseLegacyPass::runOnLoop(Loop *L, + LPPassManager &LPM) { if (skipLoop(L)) return false; + HexagonVectorLoopCarriedReuse Vlcr(L); + return Vlcr.run(); +} - if (!L->getLoopPreheader()) +bool HexagonVectorLoopCarriedReuse::run() { + if (!CurLoop->getLoopPreheader()) return false; // Work only on innermost loops. - if (!L->getSubLoops().empty()) + if (!CurLoop->getSubLoops().empty()) return false; // Work only on single basic blocks loops. - if (L->getNumBlocks() != 1) + if (CurLoop->getNumBlocks() != 1) return false; - CurLoop = L; - return doVLCR(); } @@ -745,6 +666,6 @@ void HexagonVectorLoopCarriedReuse::findLoopCarriedDeps() { ++i) { dbgs() << *Dependences[i] << "\n"; }); } -Pass *llvm::createHexagonVectorLoopCarriedReusePass() { - return new HexagonVectorLoopCarriedReuse(); +Pass *llvm::createHexagonVectorLoopCarriedReuseLegacyPass() { + return new HexagonVectorLoopCarriedReuseLegacyPass(); } diff --git a/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h new file mode 100644 index 000000000000..f1e0c5804ace --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h @@ -0,0 +1,139 @@ +//===- HexagonVectorLoopCarriedReuse.h ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass removes the computation of provably redundant expressions that have +// been computed earlier in a previous iteration. It relies on the use of PHIs +// to identify loop carried dependences. This is scalar replacement for vector +// types. +// +//----------------------------------------------------------------------------- +// Motivation: Consider the case where we have the following loop structure. +// +// Loop: +// t0 = a[i]; +// t1 = f(t0); +// t2 = g(t1); +// ... +// t3 = a[i+1]; +// t4 = f(t3); +// t5 = g(t4); +// t6 = op(t2, t5) +// cond_branch <Loop> +// +// This can be converted to +// t00 = a[0]; +// t10 = f(t00); +// t20 = g(t10); +// Loop: +// t2 = t20; +// t3 = a[i+1]; +// t4 = f(t3); +// t5 = g(t4); +// t6 = op(t2, t5) +// t20 = t5 +// cond_branch <Loop> +// +// SROA does a good job of reusing a[i+1] as a[i] in the next iteration. +// Such a loop comes to this pass in the following form. +// +// LoopPreheader: +// X0 = a[0]; +// Loop: +// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> +// t1 = f(X2) <-- I1 +// t2 = g(t1) +// ... +// X1 = a[i+1] +// t4 = f(X1) <-- I2 +// t5 = g(t4) +// t6 = op(t2, t5) +// cond_branch <Loop> +// +// In this pass, we look for PHIs such as X2 whose incoming values come only +// from the Loop Preheader and over the backedge and additionaly, both these +// values are the results of the same operation in terms of opcode. We call such +// a PHI node a dependence chain or DepChain. In this case, the dependence of X2 +// over X1 is carried over only one iteration and so the DepChain is only one +// PHI node long. +// +// Then, we traverse the uses of the PHI (X2) and the uses of the value of the +// PHI coming over the backedge (X1). We stop at the first pair of such users +// I1 (of X2) and I2 (of X1) that meet the following conditions. +// 1. I1 and I2 are the same operation, but with different operands. +// 2. X2 and X1 are used at the same operand number in the two instructions. +// 3. All other operands Op1 of I1 and Op2 of I2 are also such that there is a +// a DepChain from Op1 to Op2 of the same length as that between X2 and X1. +// +// We then make the following transformation +// LoopPreheader: +// X0 = a[0]; +// Y0 = f(X0); +// Loop: +// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> +// Y2 = PHI<(Y0, LoopPreheader), (t4, Loop)> +// t1 = f(X2) <-- Will be removed by DCE. +// t2 = g(Y2) +// ... +// X1 = a[i+1] +// t4 = f(X1) +// t5 = g(t4) +// t6 = op(t2, t5) +// cond_branch <Loop> +// +// We proceed until we cannot find any more such instructions I1 and I2. +// +// --- DepChains & Loop carried dependences --- +// Consider a single basic block loop such as +// +// LoopPreheader: +// X0 = ... +// Y0 = ... +// Loop: +// X2 = PHI<(X0, LoopPreheader), (X1, Loop)> +// Y2 = PHI<(Y0, LoopPreheader), (X2, Loop)> +// ... +// X1 = ... +// ... +// cond_branch <Loop> +// +// Then there is a dependence between X2 and X1 that goes back one iteration, +// i.e. X1 is used as X2 in the very next iteration. We represent this as a +// DepChain from X2 to X1 (X2->X1). +// Similarly, there is a dependence between Y2 and X1 that goes back two +// iterations. X1 is used as Y2 two iterations after it is computed. This is +// represented by a DepChain as (Y2->X2->X1). +// +// A DepChain has the following properties. +// 1. Num of edges in DepChain = Number of Instructions in DepChain = Number of +// iterations of carried dependence + 1. +// 2. All instructions in the DepChain except the last are PHIs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONVLCR_H +#define LLVM_LIB_TARGET_HEXAGON_HEXAGONVLCR_H + +#include "llvm/Transforms/Scalar/LoopPassManager.h" + +namespace llvm { + +class Loop; + +/// Hexagon Vector Loop Carried Reuse Pass +struct HexagonVectorLoopCarriedReusePass + : public PassInfoMixin<HexagonVectorLoopCarriedReusePass> { + HexagonVectorLoopCarriedReusePass() {} + + /// Run pass over the Loop. + PreservedAnalyses run(Loop &L, LoopAnalysisManager &LAM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONVLCR_H diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp index e7069819fa57..627c53cadd84 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp @@ -74,7 +74,7 @@ public: void setExtender(MCContext &Context) const { if (Extender == nullptr) - const_cast<HexagonAsmBackend *>(this)->Extender = new (Context) MCInst; + const_cast<HexagonAsmBackend *>(this)->Extender = Context.createMCInst(); } MCInst *takeExtender() const { @@ -736,7 +736,7 @@ public: auto &Inst = const_cast<MCInst &>(RF.getInst()); while (Size > 0 && HexagonMCInstrInfo::bundleSize(Inst) < MaxPacketSize) { - MCInst *Nop = new (Context) MCInst; + MCInst *Nop = Context.createMCInst(); Nop->setOpcode(Hexagon::A2_nop); Inst.addOperand(MCOperand::createInst(Nop)); Size -= 4; diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h index cd96a23e1b94..76658378c0cd 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h @@ -34,6 +34,7 @@ public: static char const *getRegisterName(unsigned RegNo); + std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override; void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O); void printOperand(MCInst const *MI, unsigned OpNo, raw_ostream &O) const; void printBrtarget(MCInst const *MI, unsigned OpNo, raw_ostream &O) const; diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp index 2b0bbdafa381..e7ade7834a9f 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp @@ -14,6 +14,7 @@ #include "MCTargetDesc/HexagonMCInstrInfo.h" #include "MCTargetDesc/HexagonMCShuffler.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -209,7 +210,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, case Hexagon::A2_tfrsi: Rt = L.getOperand(0); compoundOpcode = J4_jumpseti; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rt); @@ -222,7 +223,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, Rs = L.getOperand(1); compoundOpcode = J4_jumpsetr; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rt); CompoundInsn->addOperand(Rs); @@ -236,7 +237,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, Rt = L.getOperand(2); compoundOpcode = cmpeqBitOpcode[getCompoundOp(R)]; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(Rt); @@ -249,7 +250,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, Rt = L.getOperand(2); compoundOpcode = cmpgtBitOpcode[getCompoundOp(R)]; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(Rt); @@ -262,7 +263,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, Rt = L.getOperand(2); compoundOpcode = cmpgtuBitOpcode[getCompoundOp(R)]; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(Rt); @@ -280,7 +281,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, compoundOpcode = cmpeqiBitOpcode[getCompoundOp(R)]; Rs = L.getOperand(1); - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(L.getOperand(2)); @@ -298,7 +299,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, compoundOpcode = cmpgtiBitOpcode[getCompoundOp(R)]; Rs = L.getOperand(1); - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(L.getOperand(2)); @@ -309,7 +310,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, LLVM_DEBUG(dbgs() << "CX: C2_cmpgtui\n"); Rs = L.getOperand(1); compoundOpcode = cmpgtuiBitOpcode[getCompoundOp(R)]; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(L.getOperand(2)); @@ -320,7 +321,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, LLVM_DEBUG(dbgs() << "CX: S2_tstbit_i\n"); Rs = L.getOperand(1); compoundOpcode = tstBitOpcode[getCompoundOp(R)]; - CompoundInsn = new (Context) MCInst; + CompoundInsn = Context.createMCInst(); CompoundInsn->setOpcode(compoundOpcode); CompoundInsn->addOperand(Rs); CompoundInsn->addOperand(R.getOperand(1)); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp index f9f342a07f6d..fa12fe1da448 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp @@ -110,7 +110,7 @@ HexagonMCInstrInfo::bundleInstructions(MCInstrInfo const &MCII, iterator_range<MCInst::const_iterator> HexagonMCInstrInfo::bundleInstructions(MCInst const &MCI) { assert(isBundle(MCI)); - return make_range(MCI.begin() + bundleInstructionsOffset, MCI.end()); + return drop_begin(MCI, bundleInstructionsOffset); } size_t HexagonMCInstrInfo::bundleSize(MCInst const &MCI) { diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp index 7514d0e67744..5e4138ae6e09 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp @@ -468,7 +468,8 @@ MCSubtargetInfo *Hexagon_MC::createHexagonMCSubtargetInfo(const Triple &TT, StringRef CPUName = Features.first; StringRef ArchFS = Features.second; - MCSubtargetInfo *X = createHexagonMCSubtargetInfoImpl(TT, CPUName, ArchFS); + MCSubtargetInfo *X = createHexagonMCSubtargetInfoImpl( + TT, CPUName, /*TuneCPU*/ CPUName, ArchFS); if (X != nullptr && (CPUName == "hexagonv67t")) addArchSubtarget(X, ArchFS); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp index 2788b86181e2..8a44ba32606e 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp @@ -326,7 +326,7 @@ bool HexagonShuffler::ValidResourceUsage(HexagonPacketSummary const &Summary) { } // Verify the CVI slot subscriptions. - std::stable_sort(begin(), end(), HexagonInstr::lessCVI); + llvm::stable_sort(*this, HexagonInstr::lessCVI); // create vector of hvx instructions to check HVXInstsT hvxInsts; hvxInsts.clear(); @@ -609,8 +609,7 @@ llvm::Optional<HexagonShuffler::HexagonPacket> HexagonShuffler::tryAuction(HexagonPacketSummary const &Summary) const { HexagonPacket PacketResult = Packet; HexagonUnitAuction AuctionCore(Summary.ReservedSlotMask); - std::stable_sort(PacketResult.begin(), PacketResult.end(), - HexagonInstr::lessCore); + llvm::stable_sort(PacketResult, HexagonInstr::lessCore); const bool ValidSlots = llvm::all_of(insts(PacketResult), [&AuctionCore](HexagonInstr const &I) { diff --git a/llvm/lib/Target/Hexagon/RDFDeadCode.cpp b/llvm/lib/Target/Hexagon/RDFDeadCode.cpp index 5a98debd3c00..894bdf38fe17 100644 --- a/llvm/lib/Target/Hexagon/RDFDeadCode.cpp +++ b/llvm/lib/Target/Hexagon/RDFDeadCode.cpp @@ -195,8 +195,7 @@ bool DeadCodeElimination::erase(const SetVector<NodeId> &Nodes) { // If it's a code node, add all ref nodes from it. uint16_t Kind = BA.Addr->getKind(); if (Kind == NodeAttrs::Stmt || Kind == NodeAttrs::Phi) { - for (auto N : NodeAddr<CodeNode*>(BA).Addr->members(DFG)) - DRNs.push_back(N); + append_range(DRNs, NodeAddr<CodeNode*>(BA).Addr->members(DFG)); DINs.push_back(DFG.addr<InstrNode*>(I)); } else { llvm_unreachable("Unexpected code node"); |