diff options
Diffstat (limited to 'lib/CodeGen')
27 files changed, 370 insertions, 431 deletions
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 31037095aa2bb..d7995447592cc 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2033,6 +2033,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { } } // else fallthrough + LLVM_FALLTHROUGH; // The MC library also has a right-shift operator, but it isn't consistently // signed or unsigned between different targets. diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index 07ba5d36cc966..3aeb4910ab10d 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -51,7 +51,7 @@ add_llvm_library(LLVMCodeGen LiveRangeShrink.cpp LiveRegMatrix.cpp LiveRegUnits.cpp - LiveStackAnalysis.cpp + LiveStacks.cpp LiveVariables.cpp LLVMTargetMachine.cpp LocalStackSlotAllocation.cpp diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index c4794380f7912..d6f55bba716f7 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -352,8 +352,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { // Clear per function information. InsertedInsts.clear(); PromotedInsts.clear(); - BFI.reset(); - BPI.reset(); ModifiedDT = false; if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>()) { @@ -365,14 +363,16 @@ bool CodeGenPrepare::runOnFunction(Function &F) { TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); + BPI.reset(new BranchProbabilityInfo(F, *LI)); + BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); OptSize = F.optForSize(); ProfileSummaryInfo *PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); if (ProfileGuidedSectionPrefix) { - if (PSI->isFunctionHotInCallGraph(&F)) + if (PSI->isFunctionHotInCallGraph(&F, *BFI)) F.setSectionPrefix(".hot"); - else if (PSI->isFunctionColdInCallGraph(&F)) + else if (PSI->isFunctionColdInCallGraph(&F, *BFI)) F.setSectionPrefix(".unlikely"); } @@ -652,13 +652,6 @@ bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB, if (SameIncomingValueBBs.count(Pred)) return true; - if (!BFI) { - Function &F = *BB->getParent(); - LoopInfo LI{DominatorTree(F)}; - BPI.reset(new BranchProbabilityInfo(F, LI)); - BFI.reset(new BlockFrequencyInfo(F, *BPI, LI)); - } - BlockFrequency PredFreq = BFI->getBlockFreq(Pred); BlockFrequency BBFreq = BFI->getBlockFreq(BB); @@ -3704,7 +3697,7 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, } else { uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType()); if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { - ConstantOffset += CI->getSExtValue()*TypeSize; + ConstantOffset += CI->getSExtValue() * TypeSize; } else if (TypeSize) { // Scales of zero don't do anything. // We only allow one variable index at the moment. if (VariableOperand != -1) diff --git a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 87a658be4c296..a3b43c92a7fce 100644 --- a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -835,6 +835,9 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { case 64: ZeroTy = Type::getDoubleTy(Ctx); break; + case 128: + ZeroTy = Type::getFP128Ty(Ctx); + break; default: llvm_unreachable("unexpected floating-point type"); } diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 1aaf7a0ceef8d..86ce4b7a94649 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -28,7 +28,7 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRangeEdit.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 92edfb059ad65..77a7aaa957324 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -81,10 +81,9 @@ LLVMTargetMachine::LLVMTargetMachine(const Target &T, this->OptLevel = OL; } -TargetIRAnalysis LLVMTargetMachine::getTargetIRAnalysis() { - return TargetIRAnalysis([this](const Function &F) { - return TargetTransformInfo(BasicTTIImpl(this, F)); - }); +TargetTransformInfo +LLVMTargetMachine::getTargetTransformInfo(const Function &F) { + return TargetTransformInfo(BasicTTIImpl(this, F)); } /// addPassesToX helper drives creation and initialization of TargetPassConfig. diff --git a/lib/CodeGen/LiveStackAnalysis.cpp b/lib/CodeGen/LiveStacks.cpp index b0e58b0e3e5fc..80ecfdb7a507e 100644 --- a/lib/CodeGen/LiveStackAnalysis.cpp +++ b/lib/CodeGen/LiveStacks.cpp @@ -1,4 +1,4 @@ -//===-- LiveStackAnalysis.cpp - Live Stack Slot Analysis ------------------===// +//===-- LiveStacks.cpp - Live Stack Slot Analysis -------------------------===// // // The LLVM Compiler Infrastructure // @@ -13,7 +13,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" diff --git a/lib/CodeGen/MIRPrinter.cpp b/lib/CodeGen/MIRPrinter.cpp index 3568f96d2b9a0..f91cca6e4e50e 100644 --- a/lib/CodeGen/MIRPrinter.cpp +++ b/lib/CodeGen/MIRPrinter.cpp @@ -157,18 +157,14 @@ public: void print(const MachineBasicBlock &MBB); void print(const MachineInstr &MI); - void printIRBlockReference(const BasicBlock &BB); void printIRValueReference(const Value &V); void printStackObjectReference(int FrameIndex); - void printOffset(int64_t Offset); void print(const MachineInstr &MI, unsigned OpIdx, const TargetRegisterInfo *TRI, bool ShouldPrintRegisterTies, LLT TypeToPrint, bool PrintDef = true); void print(const LLVMContext &Context, const TargetInstrInfo &TII, const MachineMemOperand &Op); void printSyncScope(const LLVMContext &Context, SyncScope::ID SSID); - - void print(const MCCFIInstruction &CFI, const TargetRegisterInfo *TRI); }; } // end namespace llvm @@ -707,32 +703,6 @@ void MIPrinter::print(const MachineInstr &MI) { } } -static void printIRSlotNumber(raw_ostream &OS, int Slot) { - if (Slot == -1) - OS << "<badref>"; - else - OS << Slot; -} - -void MIPrinter::printIRBlockReference(const BasicBlock &BB) { - OS << "%ir-block."; - if (BB.hasName()) { - printLLVMNameWithoutPrefix(OS, BB.getName()); - return; - } - const Function *F = BB.getParent(); - int Slot; - if (F == MST.getCurrentFunction()) { - Slot = MST.getLocalSlot(&BB); - } else { - ModuleSlotTracker CustomMST(F->getParent(), - /*ShouldInitializeAllMetadata=*/false); - CustomMST.incorporateFunction(*F); - Slot = CustomMST.getLocalSlot(&BB); - } - printIRSlotNumber(OS, Slot); -} - void MIPrinter::printIRValueReference(const Value &V) { if (isa<GlobalValue>(V)) { V.printAsOperand(OS, /*PrintType=*/false, MST); @@ -750,7 +720,7 @@ void MIPrinter::printIRValueReference(const Value &V) { printLLVMNameWithoutPrefix(OS, V.getName()); return; } - printIRSlotNumber(OS, MST.getLocalSlot(&V)); + MachineOperand::printIRSlotNumber(OS, MST.getLocalSlot(&V)); } void MIPrinter::printStackObjectReference(int FrameIndex) { @@ -762,16 +732,6 @@ void MIPrinter::printStackObjectReference(int FrameIndex) { Operand.Name); } -void MIPrinter::printOffset(int64_t Offset) { - if (Offset == 0) - return; - if (Offset < 0) { - OS << " - " << -Offset; - return; - } - OS << " + " << Offset; -} - void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx, const TargetRegisterInfo *TRI, bool ShouldPrintRegisterTies, LLT TypeToPrint, @@ -787,6 +747,7 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx, LLVM_FALLTHROUGH; case MachineOperand::MO_Register: case MachineOperand::MO_CImmediate: + case MachineOperand::MO_FPImmediate: case MachineOperand::MO_MachineBasicBlock: case MachineOperand::MO_ConstantPoolIndex: case MachineOperand::MO_TargetIndex: @@ -795,7 +756,11 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx, case MachineOperand::MO_GlobalAddress: case MachineOperand::MO_RegisterLiveOut: case MachineOperand::MO_Metadata: - case MachineOperand::MO_MCSymbol: { + case MachineOperand::MO_MCSymbol: + case MachineOperand::MO_CFIIndex: + case MachineOperand::MO_IntrinsicID: + case MachineOperand::MO_Predicate: + case MachineOperand::MO_BlockAddress: { unsigned TiedOperandIdx = 0; if (ShouldPrintRegisterTies && Op.isReg() && Op.isTied() && !Op.isDef()) TiedOperandIdx = Op.getParent()->findTiedOperandIdx(OpIdx); @@ -804,21 +769,9 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx, TiedOperandIdx, TRI, TII); break; } - case MachineOperand::MO_FPImmediate: - Op.getFPImm()->printAsOperand(OS, /*PrintType=*/true, MST); - break; case MachineOperand::MO_FrameIndex: printStackObjectReference(Op.getIndex()); break; - case MachineOperand::MO_BlockAddress: - OS << "blockaddress("; - Op.getBlockAddress()->getFunction()->printAsOperand(OS, /*PrintType=*/false, - MST); - OS << ", "; - printIRBlockReference(*Op.getBlockAddress()->getBasicBlock()); - OS << ')'; - printOffset(Op.getOffset()); - break; case MachineOperand::MO_RegisterMask: { auto RegMaskInfo = RegisterMaskIds.find(Op.getRegMask()); if (RegMaskInfo != RegisterMaskIds.end()) @@ -827,28 +780,6 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx, printCustomRegMask(Op.getRegMask(), OS, TRI); break; } - case MachineOperand::MO_CFIIndex: { - const MachineFunction &MF = *Op.getParent()->getMF(); - print(MF.getFrameInstructions()[Op.getCFIIndex()], TRI); - break; - } - case MachineOperand::MO_IntrinsicID: { - Intrinsic::ID ID = Op.getIntrinsicID(); - if (ID < Intrinsic::num_intrinsics) - OS << "intrinsic(@" << Intrinsic::getName(ID, None) << ')'; - else { - const MachineFunction &MF = *Op.getParent()->getMF(); - const TargetIntrinsicInfo *TII = MF.getTarget().getIntrinsicInfo(); - OS << "intrinsic(@" << TII->getName(ID) << ')'; - } - break; - } - case MachineOperand::MO_Predicate: { - auto Pred = static_cast<CmpInst::Predicate>(Op.getPredicate()); - OS << (CmpInst::isIntPredicate(Pred) ? "int" : "float") << "pred(" - << CmpInst::getPredicateName(Pred) << ')'; - break; - } } } @@ -938,7 +869,7 @@ void MIPrinter::print(const LLVMContext &Context, const TargetInstrInfo &TII, break; } } - printOffset(Op.getOffset()); + MachineOperand::printOperandOffset(OS, Op.getOffset()); if (Op.getBaseAlignment() != Op.getSize()) OS << ", align " << Op.getBaseAlignment(); auto AAInfo = Op.getAAInfo(); @@ -978,118 +909,6 @@ void MIPrinter::printSyncScope(const LLVMContext &Context, SyncScope::ID SSID) { } } -static void printCFIRegister(unsigned DwarfReg, raw_ostream &OS, - const TargetRegisterInfo *TRI) { - int Reg = TRI->getLLVMRegNum(DwarfReg, true); - if (Reg == -1) { - OS << "<badreg>"; - return; - } - OS << printReg(Reg, TRI); -} - -void MIPrinter::print(const MCCFIInstruction &CFI, - const TargetRegisterInfo *TRI) { - switch (CFI.getOperation()) { - case MCCFIInstruction::OpSameValue: - OS << "same_value "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - break; - case MCCFIInstruction::OpRememberState: - OS << "remember_state "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - break; - case MCCFIInstruction::OpRestoreState: - OS << "restore_state "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - break; - case MCCFIInstruction::OpOffset: - OS << "offset "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - OS << ", " << CFI.getOffset(); - break; - case MCCFIInstruction::OpDefCfaRegister: - OS << "def_cfa_register "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - break; - case MCCFIInstruction::OpDefCfaOffset: - OS << "def_cfa_offset "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - OS << CFI.getOffset(); - break; - case MCCFIInstruction::OpDefCfa: - OS << "def_cfa "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - OS << ", " << CFI.getOffset(); - break; - case MCCFIInstruction::OpRelOffset: - OS << "rel_offset "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - OS << ", " << CFI.getOffset(); - break; - case MCCFIInstruction::OpAdjustCfaOffset: - OS << "adjust_cfa_offset "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - OS << CFI.getOffset(); - break; - case MCCFIInstruction::OpRestore: - OS << "restore "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - break; - case MCCFIInstruction::OpEscape: { - OS << "escape "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - if (!CFI.getValues().empty()) { - size_t e = CFI.getValues().size() - 1; - for (size_t i = 0; i < e; ++i) - OS << format("0x%02x", uint8_t(CFI.getValues()[i])) << ", "; - OS << format("0x%02x", uint8_t(CFI.getValues()[e])) << ", "; - } - break; - } - case MCCFIInstruction::OpUndefined: - OS << "undefined "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - break; - case MCCFIInstruction::OpRegister: - OS << "register "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - printCFIRegister(CFI.getRegister(), OS, TRI); - OS << ", "; - printCFIRegister(CFI.getRegister2(), OS, TRI); - break; - case MCCFIInstruction::OpWindowSave: - OS << "window_save "; - if (MCSymbol *Label = CFI.getLabel()) - MachineOperand::printSymbol(OS, *Label); - break; - default: - // TODO: Print the other CFI Operations. - OS << "<unserializable cfi operation>"; - break; - } -} - void llvm::printMIR(raw_ostream &OS, const Module &M) { yaml::Output Out(OS); Out << const_cast<Module &>(M); diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp index 4ce6896077303..84c808ee79384 100644 --- a/lib/CodeGen/MachineBlockPlacement.cpp +++ b/lib/CodeGen/MachineBlockPlacement.cpp @@ -1235,7 +1235,7 @@ void MachineBlockPlacement::precomputeTriangleChains() { // When profile is available, we need to handle the triangle-shape CFG. static BranchProbability getLayoutSuccessorProbThreshold( const MachineBasicBlock *BB) { - if (!BB->getParent()->getFunction().getEntryCount()) + if (!BB->getParent()->getFunction().hasProfileData()) return BranchProbability(StaticLikelyProb, 100); if (BB->succ_size() == 2) { const MachineBasicBlock *Succ1 = *BB->succ_begin(); @@ -2178,7 +2178,7 @@ MachineBlockPlacement::collectLoopBlockSet(const MachineLoop &L) { // will be merged into the first outer loop chain for which this block is not // cold anymore. This needs precise profile data and we only do this when // profile data is available. - if (F->getFunction().getEntryCount() || ForceLoopColdBlock) { + if (F->getFunction().hasProfileData() || ForceLoopColdBlock) { BlockFrequency LoopFreq(0); for (auto LoopPred : L.getHeader()->predecessors()) if (!L.contains(LoopPred)) @@ -2220,7 +2220,7 @@ void MachineBlockPlacement::buildLoopChains(const MachineLoop &L) { // for better layout. bool RotateLoopWithProfile = ForcePreciseRotationCost || - (PreciseRotationCost && F->getFunction().getEntryCount()); + (PreciseRotationCost && F->getFunction().hasProfileData()); // First check to see if there is an obviously preferable top block for the // loop. This will default to the header, but may end up as one of the diff --git a/lib/CodeGen/MachineOperand.cpp b/lib/CodeGen/MachineOperand.cpp index d17c481862a1b..ec81c63911712 100644 --- a/lib/CodeGen/MachineOperand.cpp +++ b/lib/CodeGen/MachineOperand.cpp @@ -380,16 +380,6 @@ static void tryToGetTargetInfo(const MachineOperand &MO, } } -static void printOffset(raw_ostream &OS, int64_t Offset) { - if (Offset == 0) - return; - if (Offset < 0) { - OS << " - " << -Offset; - return; - } - OS << " + " << Offset; -} - static const char *getTargetIndexName(const MachineFunction &MF, int Index) { const auto *TII = MF.getSubtarget().getInstrInfo(); assert(TII && "expected instruction info"); @@ -412,6 +402,44 @@ static const char *getTargetFlagName(const TargetInstrInfo *TII, unsigned TF) { return nullptr; } +static void printCFIRegister(unsigned DwarfReg, raw_ostream &OS, + const TargetRegisterInfo *TRI) { + if (!TRI) { + OS << "%dwarfreg." << DwarfReg; + return; + } + + int Reg = TRI->getLLVMRegNum(DwarfReg, true); + if (Reg == -1) { + OS << "<badreg>"; + return; + } + OS << printReg(Reg, TRI); +} + +static void printIRBlockReference(raw_ostream &OS, const BasicBlock &BB, + ModuleSlotTracker &MST) { + OS << "%ir-block."; + if (BB.hasName()) { + printLLVMNameWithoutPrefix(OS, BB.getName()); + return; + } + Optional<int> Slot; + if (const Function *F = BB.getParent()) { + if (F == MST.getCurrentFunction()) { + Slot = MST.getLocalSlot(&BB); + } else if (const Module *M = F->getParent()) { + ModuleSlotTracker CustomMST(M, /*ShouldInitializeAllMetadata=*/false); + CustomMST.incorporateFunction(*F); + Slot = CustomMST.getLocalSlot(&BB); + } + } + if (Slot) + MachineOperand::printIRSlotNumber(OS, *Slot); + else + OS << "<unknown>"; +} + void MachineOperand::printSubregIdx(raw_ostream &OS, uint64_t Index, const TargetRegisterInfo *TRI) { OS << "%subreg."; @@ -490,6 +518,125 @@ void MachineOperand::printStackObjectReference(raw_ostream &OS, OS << '.' << Name; } +void MachineOperand::printOperandOffset(raw_ostream &OS, int64_t Offset) { + if (Offset == 0) + return; + if (Offset < 0) { + OS << " - " << -Offset; + return; + } + OS << " + " << Offset; +} + +void MachineOperand::printIRSlotNumber(raw_ostream &OS, int Slot) { + if (Slot == -1) + OS << "<badref>"; + else + OS << Slot; +} + +static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI, + const TargetRegisterInfo *TRI) { + switch (CFI.getOperation()) { + case MCCFIInstruction::OpSameValue: + OS << "same_value "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + break; + case MCCFIInstruction::OpRememberState: + OS << "remember_state "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + break; + case MCCFIInstruction::OpRestoreState: + OS << "restore_state "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + break; + case MCCFIInstruction::OpOffset: + OS << "offset "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + OS << ", " << CFI.getOffset(); + break; + case MCCFIInstruction::OpDefCfaRegister: + OS << "def_cfa_register "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + break; + case MCCFIInstruction::OpDefCfaOffset: + OS << "def_cfa_offset "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + OS << CFI.getOffset(); + break; + case MCCFIInstruction::OpDefCfa: + OS << "def_cfa "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + OS << ", " << CFI.getOffset(); + break; + case MCCFIInstruction::OpRelOffset: + OS << "rel_offset "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + OS << ", " << CFI.getOffset(); + break; + case MCCFIInstruction::OpAdjustCfaOffset: + OS << "adjust_cfa_offset "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + OS << CFI.getOffset(); + break; + case MCCFIInstruction::OpRestore: + OS << "restore "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + break; + case MCCFIInstruction::OpEscape: { + OS << "escape "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + if (!CFI.getValues().empty()) { + size_t e = CFI.getValues().size() - 1; + for (size_t i = 0; i < e; ++i) + OS << format("0x%02x", uint8_t(CFI.getValues()[i])) << ", "; + OS << format("0x%02x", uint8_t(CFI.getValues()[e])) << ", "; + } + break; + } + case MCCFIInstruction::OpUndefined: + OS << "undefined "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + break; + case MCCFIInstruction::OpRegister: + OS << "register "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + printCFIRegister(CFI.getRegister(), OS, TRI); + OS << ", "; + printCFIRegister(CFI.getRegister2(), OS, TRI); + break; + case MCCFIInstruction::OpWindowSave: + OS << "window_save "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + break; + default: + // TODO: Print the other CFI Operations. + OS << "<unserializable cfi directive>"; + break; + } +} + void MachineOperand::print(raw_ostream &OS, const TargetRegisterInfo *TRI, const TargetIntrinsicInfo *IntrinsicInfo) const { tryToGetTargetInfo(*this, TRI, IntrinsicInfo); @@ -561,29 +708,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, getCImm()->printAsOperand(OS, /*PrintType=*/true, MST); break; case MachineOperand::MO_FPImmediate: - if (getFPImm()->getType()->isFloatTy()) { - OS << getFPImm()->getValueAPF().convertToFloat(); - } else if (getFPImm()->getType()->isHalfTy()) { - APFloat APF = getFPImm()->getValueAPF(); - bool Unused; - APF.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, &Unused); - OS << "half " << APF.convertToFloat(); - } else if (getFPImm()->getType()->isFP128Ty()) { - APFloat APF = getFPImm()->getValueAPF(); - SmallString<16> Str; - getFPImm()->getValueAPF().toString(Str); - OS << "quad " << Str; - } else if (getFPImm()->getType()->isX86_FP80Ty()) { - APFloat APF = getFPImm()->getValueAPF(); - OS << "x86_fp80 0xK"; - APInt API = APF.bitcastToAPInt(); - OS << format_hex_no_prefix(API.getHiBits(16).getZExtValue(), 4, - /*Upper=*/true); - OS << format_hex_no_prefix(API.getLoBits(64).getZExtValue(), 16, - /*Upper=*/true); - } else { - OS << getFPImm()->getValueAPF().convertToDouble(); - } + getFPImm()->printAsOperand(OS, /*PrintType=*/true, MST); break; case MachineOperand::MO_MachineBasicBlock: OS << printMBBReference(*getMBB()); @@ -606,7 +731,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, } case MachineOperand::MO_ConstantPoolIndex: OS << "%const." << getIndex(); - printOffset(OS, getOffset()); + printOperandOffset(OS, getOffset()); break; case MachineOperand::MO_TargetIndex: { OS << "target-index("; @@ -615,7 +740,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, if (const auto *TargetIndexName = getTargetIndexName(*MF, getIndex())) Name = TargetIndexName; OS << Name << ')'; - printOffset(OS, getOffset()); + printOperandOffset(OS, getOffset()); break; } case MachineOperand::MO_JumpTableIndex: @@ -623,7 +748,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, break; case MachineOperand::MO_GlobalAddress: getGlobal()->printAsOperand(OS, /*PrintType=*/false, MST); - printOffset(OS, getOffset()); + printOperandOffset(OS, getOffset()); break; case MachineOperand::MO_ExternalSymbol: { StringRef Name = getSymbolName(); @@ -633,16 +758,19 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, } else { printLLVMNameWithoutPrefix(OS, Name); } - printOffset(OS, getOffset()); + printOperandOffset(OS, getOffset()); break; } - case MachineOperand::MO_BlockAddress: - OS << '<'; - getBlockAddress()->printAsOperand(OS, /*PrintType=*/false, MST); - if (getOffset()) - OS << "+" << getOffset(); - OS << '>'; + case MachineOperand::MO_BlockAddress: { + OS << "blockaddress("; + getBlockAddress()->getFunction()->printAsOperand(OS, /*PrintType=*/false, + MST); + OS << ", "; + printIRBlockReference(OS, *getBlockAddress()->getBasicBlock(), MST); + OS << ')'; + MachineOperand::printOperandOffset(OS, getOffset()); break; + } case MachineOperand::MO_RegisterMask: { OS << "<regmask"; if (TRI) { @@ -693,23 +821,27 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, case MachineOperand::MO_MCSymbol: printSymbol(OS, *getMCSymbol()); break; - case MachineOperand::MO_CFIIndex: - OS << "<call frame instruction>"; + case MachineOperand::MO_CFIIndex: { + if (const MachineFunction *MF = getMFIfAvailable(*this)) + printCFI(OS, MF->getFrameInstructions()[getCFIIndex()], TRI); + else + OS << "<cfi directive>"; break; + } case MachineOperand::MO_IntrinsicID: { Intrinsic::ID ID = getIntrinsicID(); if (ID < Intrinsic::num_intrinsics) - OS << "<intrinsic:@" << Intrinsic::getName(ID, None) << '>'; + OS << "intrinsic(@" << Intrinsic::getName(ID, None) << ')'; else if (IntrinsicInfo) - OS << "<intrinsic:@" << IntrinsicInfo->getName(ID) << '>'; + OS << "intrinsic(@" << IntrinsicInfo->getName(ID) << ')'; else - OS << "<intrinsic:" << ID << '>'; + OS << "intrinsic(" << ID << ')'; break; } case MachineOperand::MO_Predicate: { auto Pred = static_cast<CmpInst::Predicate>(getPredicate()); - OS << '<' << (CmpInst::isIntPredicate(Pred) ? "intpred" : "floatpred") - << CmpInst::getPredicateName(Pred) << '>'; + OS << (CmpInst::isIntPredicate(Pred) ? "int" : "float") << "pred(" + << CmpInst::getPredicateName(Pred) << ')'; break; } } diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index c9fe7681e2805..e0cc2ca9a2a29 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -37,7 +37,7 @@ #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" diff --git a/lib/CodeGen/README.txt b/lib/CodeGen/README.txt index 2fcbd1280da42..3318e109155b8 100644 --- a/lib/CodeGen/README.txt +++ b/lib/CodeGen/README.txt @@ -164,7 +164,7 @@ synthesize the various copy insertion/inspection methods in TargetInstrInfo. Stack coloring improvements: -1. Do proper LiveStackAnalysis on all stack objects including those which are +1. Do proper LiveStacks analysis on all stack objects including those which are not spill slots. 2. Reorder objects to fill in gaps between objects. e.g. 4, 1, <gap>, 4, 1, 1, 1, <gap>, 4 => 4, 1, 1, 1, 1, 4, 4 diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index 6e273277804bf..1125d2c62bef3 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -21,7 +21,7 @@ #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveRegMatrix.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index 186ef577e31df..e492c481a540e 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -39,7 +39,7 @@ #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveRegMatrix.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 351e91c932eb7..69a879701fae2 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -45,7 +45,7 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRangeEdit.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f97732c1c49d0..17f907eb07e8d 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3988,10 +3988,12 @@ SDValue DAGCombiner::visitAND(SDNode *N) { if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1)) return RAND; // fold (and (or x, C), D) -> D if (C & D) == D - if (N1C && N0.getOpcode() == ISD::OR) - if (ConstantSDNode *ORI = isConstOrConstSplat(N0.getOperand(1))) - if (N1C->getAPIntValue().isSubsetOf(ORI->getAPIntValue())) - return N1; + auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) { + return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue()); + }; + if (N0.getOpcode() == ISD::OR && + matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset)) + return N1; // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits. if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { SDValue N0Op0 = N0.getOperand(0); @@ -4675,16 +4677,16 @@ SDValue DAGCombiner::visitOR(SDNode *N) { // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) // iff (c1 & c2) != 0. - if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse()) { - if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { - if (C1->getAPIntValue().intersects(N1C->getAPIntValue())) { - if (SDValue COR = - DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, N1C, C1)) - return DAG.getNode( - ISD::AND, SDLoc(N), VT, - DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1), COR); - return SDValue(); - } + auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) { + return LHS->getAPIntValue().intersects(RHS->getAPIntValue()); + }; + if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && + matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) { + if (SDValue COR = DAG.FoldConstantArithmetic( + ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) { + SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); + AddToWorklist(IOR.getNode()); + return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); } } @@ -5380,21 +5382,6 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { AddToWorklist(NotX.getNode()); return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1); } - // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2)) - if (N1C && N0.getOpcode() == ISD::XOR) { - if (const ConstantSDNode *N00C = getAsNonOpaqueConstant(N0.getOperand(0))) { - SDLoc DL(N); - return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), - DAG.getConstant(N1C->getAPIntValue() ^ - N00C->getAPIntValue(), DL, VT)); - } - if (const ConstantSDNode *N01C = getAsNonOpaqueConstant(N0.getOperand(1))) { - SDLoc DL(N); - return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), - DAG.getConstant(N1C->getAPIntValue() ^ - N01C->getAPIntValue(), DL, VT)); - } - } // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X) unsigned OpSizeInBits = VT.getScalarSizeInBits(); @@ -10201,7 +10188,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { case ISD::SETLT: case ISD::SETLE: std::swap(TrueOpnd, FalseOpnd); - // Fall through + LLVM_FALLTHROUGH; case ISD::SETOGT: case ISD::SETUGT: case ISD::SETOGE: @@ -10555,7 +10542,7 @@ static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N) { // value in one SSE register, but instruction selection cannot handle // FCOPYSIGN on SSE registers yet. EVT N1VT = N1->getValueType(0); - EVT N1Op0VT = N1->getOperand(0)->getValueType(0); + EVT N1Op0VT = N1->getOperand(0).getValueType(); return (N1VT == N1Op0VT || N1Op0VT != MVT::f128); } return false; @@ -13784,30 +13771,30 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { } } - if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) { - if (ST->isUnindexed() && !ST->isVolatile() && ST1->isUnindexed() && - !ST1->isVolatile() && ST1->getBasePtr() == Ptr && - ST->getMemoryVT() == ST1->getMemoryVT()) { - // If this is a store followed by a store with the same value to the same - // location, then the store is dead/noop. - if (ST1->getValue() == Value) { - // The store is dead, remove it. - return Chain; - } - - // If this is a store who's preceeding store to the same location - // and no one other node is chained to that store we can effectively - // drop the store. Do not remove stores to undef as they may be used as - // data sinks. - if (OptLevel != CodeGenOpt::None && ST1->hasOneUse() && - !ST1->getBasePtr().isUndef()) { - // ST1 is fully overwritten and can be elided. Combine with it's chain - // value. + // Deal with elidable overlapping chained stores. + if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) + if (OptLevel != CodeGenOpt::None && ST->isUnindexed() && + ST1->isUnindexed() && !ST1->isVolatile() && ST1->hasOneUse() && + !ST1->getBasePtr().isUndef() && !ST->isVolatile()) { + BaseIndexOffset STBasePtr = BaseIndexOffset::match(ST->getBasePtr(), DAG); + BaseIndexOffset ST1BasePtr = + BaseIndexOffset::match(ST1->getBasePtr(), DAG); + unsigned STBytes = ST->getMemoryVT().getStoreSize(); + unsigned ST1Bytes = ST1->getMemoryVT().getStoreSize(); + int64_t PtrDiff; + // If this is a store who's preceeding store to a subset of the same + // memory and no one other node is chained to that store we can + // effectively drop the store. Do not remove stores to undef as they may + // be used as data sinks. + + if (((ST->getBasePtr() == ST1->getBasePtr()) && + (ST->getValue() == ST1->getValue())) || + (STBasePtr.equalBaseIndex(ST1BasePtr, DAG, PtrDiff) && + (0 <= PtrDiff) && (PtrDiff + ST1Bytes <= STBytes))) { CombineTo(ST1, ST1->getChain()); - return SDValue(); + return SDValue(N, 0); } } - } // If this is an FP_ROUND or TRUNC followed by a store, fold this into a // truncating store. We can do this even if this is already a truncstore. @@ -15110,7 +15097,7 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { // Transform: concat_vectors(scalar, undef) -> scalar_to_vector(sclr). if (In->getOpcode() == ISD::BITCAST && - !In->getOperand(0)->getValueType(0).isVector()) { + !In->getOperand(0).getValueType().isVector()) { SDValue Scalar = In->getOperand(0); // If the bitcast type isn't legal, it might be a trunc of a legal type; @@ -15157,7 +15144,7 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { bool FoundMinVT = false; for (const SDValue &Op : N->ops()) if (ISD::BUILD_VECTOR == Op.getOpcode()) { - EVT OpSVT = Op.getOperand(0)->getValueType(0); + EVT OpSVT = Op.getOperand(0).getValueType(); MinVT = (!FoundMinVT || OpSVT.bitsLE(MinVT)) ? OpSVT : MinVT; FoundMinVT = true; } @@ -17418,43 +17405,6 @@ SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) { return buildSqrtEstimateImpl(Op, Flags, false); } -/// Return true if base is a frame index, which is known not to alias with -/// anything but itself. Provides base object and offset as results. -static bool findBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, - const GlobalValue *&GV, const void *&CV) { - // Assume it is a primitive operation. - Base = Ptr; Offset = 0; GV = nullptr; CV = nullptr; - - // If it's an adding a simple constant then integrate the offset. - if (Base.getOpcode() == ISD::ADD) { - if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) { - Base = Base.getOperand(0); - Offset += C->getSExtValue(); - } - } - - // Return the underlying GlobalValue, and update the Offset. Return false - // for GlobalAddressSDNode since the same GlobalAddress may be represented - // by multiple nodes with different offsets. - if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Base)) { - GV = G->getGlobal(); - Offset += G->getOffset(); - return false; - } - - // Return the underlying Constant value, and update the Offset. Return false - // for ConstantSDNodes since the same constant pool entry may be represented - // by multiple nodes with different offsets. - if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) { - CV = C->isMachineConstantPoolEntry() ? (const void *)C->getMachineCPVal() - : (const void *)C->getConstVal(); - Offset += C->getOffset(); - return false; - } - // If it's any of the following then it can't alias with anything but itself. - return isa<FrameIndexSDNode>(Base); -} - /// Return true if there is any possibility that the two addresses overlap. bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { // If they are the same then they must be aliases. @@ -17496,39 +17446,18 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { return false; } - // FIXME: findBaseOffset and ConstantValue/GlobalValue/FrameIndex analysis - // modified to use BaseIndexOffset. - - // Gather base node and offset information. - SDValue Base0, Base1; - int64_t Offset0, Offset1; - const GlobalValue *GV0, *GV1; - const void *CV0, *CV1; - bool IsFrameIndex0 = findBaseOffset(Op0->getBasePtr(), - Base0, Offset0, GV0, CV0); - bool IsFrameIndex1 = findBaseOffset(Op1->getBasePtr(), - Base1, Offset1, GV1, CV1); - - // If they have the same base address, then check to see if they overlap. - if (Base0 == Base1 || (GV0 && (GV0 == GV1)) || (CV0 && (CV0 == CV1))) - return !((Offset0 + NumBytes0) <= Offset1 || - (Offset1 + NumBytes1) <= Offset0); - - // It is possible for different frame indices to alias each other, mostly - // when tail call optimization reuses return address slots for arguments. - // To catch this case, look up the actual index of frame indices to compute - // the real alias relationship. - if (IsFrameIndex0 && IsFrameIndex1) { - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - Offset0 += MFI.getObjectOffset(cast<FrameIndexSDNode>(Base0)->getIndex()); - Offset1 += MFI.getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex()); - return !((Offset0 + NumBytes0) <= Offset1 || - (Offset1 + NumBytes1) <= Offset0); - } - - // Otherwise, if we know what the bases are, and they aren't identical, then - // we know they cannot alias. - if ((IsFrameIndex0 || CV0 || GV0) && (IsFrameIndex1 || CV1 || GV1)) + bool IsFI0 = isa<FrameIndexSDNode>(BasePtr0.getBase()); + bool IsFI1 = isa<FrameIndexSDNode>(BasePtr1.getBase()); + bool IsGV0 = isa<GlobalAddressSDNode>(BasePtr0.getBase()); + bool IsGV1 = isa<GlobalAddressSDNode>(BasePtr1.getBase()); + bool IsCV0 = isa<ConstantPoolSDNode>(BasePtr0.getBase()); + bool IsCV1 = isa<ConstantPoolSDNode>(BasePtr1.getBase()); + + // If of mismatched base types or checkable indices we can check + // they do not alias. + if ((BasePtr0.getIndex() == BasePtr1.getIndex() || (IsFI0 != IsFI1) || + (IsGV0 != IsGV1) || (IsCV0 != IsCV1)) && + (IsFI0 || IsGV0 || IsCV0) && (IsFI1 || IsGV1 || IsCV1)) return false; // If we know required SrcValue1 and SrcValue2 have relatively large alignment diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index eaf177d0661b3..e28a3aa47ca3c 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -1887,7 +1887,7 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { SDLoc DL(N); SDValue Promoted = GetPromotedFloat(Val); - EVT VT = ST->getOperand(1)->getValueType(0); + EVT VT = ST->getOperand(1).getValueType(); EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); SDValue NewVal; diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index b60d7bca498ad..4438ee7878b8c 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -224,7 +224,7 @@ bool DAGTypeLegalizer::run() { assert(N->getNodeId() == ReadyToProcess && "Node should be ready if on worklist!"); - DEBUG(dbgs() << "Legalizing node: "; N->dump()); + DEBUG(dbgs() << "Legalizing node: "; N->dump(&DAG)); if (IgnoreNodeResults(N)) { DEBUG(dbgs() << "Ignoring node results\n"); goto ScanOperands; @@ -296,7 +296,7 @@ ScanOperands: continue; const auto Op = N->getOperand(i); - DEBUG(dbgs() << "Analyzing operand: "; Op.dump()); + DEBUG(dbgs() << "Analyzing operand: "; Op.dump(&DAG)); EVT OpVT = Op.getValueType(); switch (getTypeAction(OpVT)) { case TargetLowering::TypeLegal: @@ -445,7 +445,7 @@ NodeDone: if (!isTypeLegal(Node.getValueType(i)) && !TLI.isTypeLegal(Node.getValueType(i))) { dbgs() << "Result type " << i << " illegal: "; - Node.dump(); + Node.dump(&DAG); Failed = true; } @@ -455,7 +455,7 @@ NodeDone: !isTypeLegal(Node.getOperand(i).getValueType()) && !TLI.isTypeLegal(Node.getOperand(i).getValueType())) { dbgs() << "Operand type " << i << " illegal: "; - Node.getOperand(i).dump(); + Node.getOperand(i).dump(&DAG); Failed = true; } diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 8f2320f52a0f3..ce1c01b621f05 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -331,7 +331,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { // At least try the common case where the boolean is generated by a // comparison. if (Cond->getOpcode() == ISD::SETCC) { - EVT OpVT = Cond->getOperand(0)->getValueType(0); + EVT OpVT = Cond->getOperand(0).getValueType(); ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); VecBool = TLI.getBooleanContents(OpVT); } else @@ -1548,14 +1548,14 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: - if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) + if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType())) Res = SplitVecOp_TruncateHelper(N); else Res = SplitVecOp_UnaryOp(N); break; case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: - if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) + if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType())) Res = SplitVecOp_TruncateHelper(N); else Res = SplitVecOp_UnaryOp(N); diff --git a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp index 379f0dcef513d..7f369c746d24e 100644 --- a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp +++ b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp @@ -252,6 +252,7 @@ bool ResourcePriorityQueue::isResourceAvailable(SUnit *SU) { if (!ResourcesModel->canReserveResources(&TII->get( SU->getNode()->getMachineOpcode()))) return false; + break; case TargetOpcode::EXTRACT_SUBREG: case TargetOpcode::INSERT_SUBREG: case TargetOpcode::SUBREG_TO_REG: diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 12a21e74079ec..a04c770c51c4d 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3750,6 +3750,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::TRUNCATE: + case ISD::ANY_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::SIGN_EXTEND: case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: case ISD::ABS: diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 544da362be698..d5980919d03c0 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -37,6 +37,23 @@ bool BaseIndexOffset::equalBaseIndex(BaseIndexOffset &Other, return true; } + // Match Constants + if (auto *A = dyn_cast<ConstantPoolSDNode>(Base)) + if (auto *B = dyn_cast<ConstantPoolSDNode>(Other.Base)) { + bool IsMatch = + A->isMachineConstantPoolEntry() == B->isMachineConstantPoolEntry(); + if (IsMatch) { + if (A->isMachineConstantPoolEntry()) + IsMatch = A->getMachineCPVal() == B->getMachineCPVal(); + else + IsMatch = A->getConstVal() == B->getConstVal(); + } + if (IsMatch) { + Off += B->getOffset() - A->getOffset(); + return true; + } + } + const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); // Match non-equal FrameIndexes - If both frame indices are fixed diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 18f6997ef83c3..d13ccc2637183 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3117,7 +3117,16 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; } case OPC_RecordMemRef: - MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand()); + if (auto *MN = dyn_cast<MemSDNode>(N)) + MatchedMemRefs.push_back(MN->getMemOperand()); + else { + DEBUG( + dbgs() << "Expected MemSDNode "; + N->dump(CurDAG); + dbgs() << '\n' + ); + } + continue; case OPC_CaptureGlueInput: @@ -3563,7 +3572,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, Ops.push_back(InputGlue); // Create the node. - SDNode *Res = nullptr; + MachineSDNode *Res = nullptr; bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo || (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2); if (!IsMorphNodeTo) { @@ -3589,7 +3598,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, "Chain node replaced during MorphNode"); Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end()); }); - Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo); + Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList, + Ops, EmitNodeInfo)); } // If the node had chain/glue results, update our notion of the current @@ -3645,13 +3655,19 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, } } - cast<MachineSDNode>(Res) - ->setMemRefs(MemRefs, MemRefs + NumMemRefs); + Res->setMemRefs(MemRefs, MemRefs + NumMemRefs); } - DEBUG(dbgs() << " " - << (IsMorphNodeTo ? "Morphed" : "Created") - << " node: "; Res->dump(CurDAG); dbgs() << "\n"); + DEBUG( + if (!MatchedMemRefs.empty() && Res->memoperands_empty()) + dbgs() << " Dropping mem operands\n"; + dbgs() << " " + << (IsMorphNodeTo ? "Morphed" : "Created") + << " node: "; + Res->dump(CurDAG); + + dbgs() << '\n'; + ); // If this was a MorphNodeTo then we're completely done! if (IsMorphNodeTo) { diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 58276052c10bd..d76e52d788703 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3812,7 +3812,7 @@ SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG, Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index, DAG.getConstant(EltSize, dl, IdxVT)); - return DAG.getNode(ISD::ADD, dl, IdxVT, Index, VecPtr); + return DAG.getNode(ISD::ADD, dl, IdxVT, VecPtr, Index); } //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index 62f662d1ade45..8fc7a4a32842c 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -16,7 +16,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 543c12eebb458..224ae1a3236aa 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -89,6 +89,21 @@ static cl::opt<unsigned> OptsizeJumpTableDensity( cl::desc("Minimum density for building a jump table in " "an optsize function")); +static bool darwinHasSinCos(const Triple &TT) { + assert(TT.isOSDarwin() && "should be called with darwin triple"); + // Don't bother with 32 bit x86. + if (TT.getArch() == Triple::x86) + return false; + // Macos < 10.9 has no sincos_stret. + if (TT.isMacOSX()) + return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit(); + // iOS < 7.0 has no sincos_stret. + if (TT.isiOS()) + return !TT.isOSVersionLT(7, 0); + // Any other darwin such as WatchOS/TvOS is new enough. + return true; +} + // Although this default value is arbitrary, it is not random. It is assumed // that a condition that evaluates the same way by a higher percentage than this // is best represented as control flow. Therefore, the default value N should be @@ -100,44 +115,56 @@ static cl::opt<int> MinPercentageForPredictableBranch( "or false to assume that the condition is predictable"), cl::Hidden); -/// InitLibcallNames - Set default libcall names. -static void InitLibcallNames(const char **Names, const Triple &TT) { +void TargetLoweringBase::InitLibcalls(const Triple &TT) { #define HANDLE_LIBCALL(code, name) \ - Names[RTLIB::code] = name; + setLibcallName(RTLIB::code, name); #include "llvm/CodeGen/RuntimeLibcalls.def" #undef HANDLE_LIBCALL + // Initialize calling conventions to their default. + for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC) + setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C); // A few names are different on particular architectures or environments. if (TT.isOSDarwin()) { // For f16/f32 conversions, Darwin uses the standard naming scheme, instead // of the gnueabi-style __gnu_*_ieee. // FIXME: What about other targets? - Names[RTLIB::FPEXT_F16_F32] = "__extendhfsf2"; - Names[RTLIB::FPROUND_F32_F16] = "__truncsfhf2"; + setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); + setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); + + // Darwin 10 and higher has an optimized __bzero. + if (!TT.isMacOSX() || !TT.isMacOSXVersionLT(10, 6) || TT.isArch64Bit()) { + setLibcallName(RTLIB::BZERO, TT.isAArch64() ? "bzero" : "__bzero"); + } + + if (darwinHasSinCos(TT)) { + setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret"); + setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret"); + if (TT.isWatchABI()) { + setLibcallCallingConv(RTLIB::SINCOS_STRET_F32, + CallingConv::ARM_AAPCS_VFP); + setLibcallCallingConv(RTLIB::SINCOS_STRET_F64, + CallingConv::ARM_AAPCS_VFP); + } + } } else { - Names[RTLIB::FPEXT_F16_F32] = "__gnu_h2f_ieee"; - Names[RTLIB::FPROUND_F32_F16] = "__gnu_f2h_ieee"; + setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee"); + setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee"); } if (TT.isGNUEnvironment() || TT.isOSFuchsia()) { - Names[RTLIB::SINCOS_F32] = "sincosf"; - Names[RTLIB::SINCOS_F64] = "sincos"; - Names[RTLIB::SINCOS_F80] = "sincosl"; - Names[RTLIB::SINCOS_F128] = "sincosl"; - Names[RTLIB::SINCOS_PPCF128] = "sincosl"; + setLibcallName(RTLIB::SINCOS_F32, "sincosf"); + setLibcallName(RTLIB::SINCOS_F64, "sincos"); + setLibcallName(RTLIB::SINCOS_F80, "sincosl"); + setLibcallName(RTLIB::SINCOS_F128, "sincosl"); + setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl"); } if (TT.isOSOpenBSD()) { - Names[RTLIB::STACKPROTECTOR_CHECK_FAIL] = nullptr; + setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr); } } -/// Set default libcall CallingConvs. -static void InitLibcallCallingConvs(CallingConv::ID *CCs) { - for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC) - CCs[LC] = CallingConv::C; -} - /// getFPEXT - Return the FPEXT_*_* value for the given types, or /// UNKNOWN_LIBCALL if there is none. RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { @@ -524,9 +551,8 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr); - InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple()); + InitLibcalls(TM.getTargetTriple()); InitCmpLibcallCCs(CmpLibcallCCs); - InitLibcallCallingConvs(LibcallCallingConvs); } void TargetLoweringBase::initActions() { diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp index 64bb37a280a62..13f7e83f3dd08 100644 --- a/lib/CodeGen/VirtRegMap.cpp +++ b/lib/CodeGen/VirtRegMap.cpp @@ -22,7 +22,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" |