From d8e91e46262bc44006913e6796843909f1ac7bcd Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Sat, 19 Jan 2019 10:01:25 +0000 Subject: Vendor import of llvm trunk r351319 (just before the release_80 branch point): https://llvm.org/svn/llvm-project/llvm/trunk@351319 --- include/llvm/CodeGen/AsmPrinter.h | 16 +- include/llvm/CodeGen/AsmPrinterHandler.h | 74 +++ include/llvm/CodeGen/BasicTTIImpl.h | 253 +++++++-- include/llvm/CodeGen/BuiltinGCs.h | 33 ++ include/llvm/CodeGen/CommandFlags.inc | 28 +- include/llvm/CodeGen/DbgEntityHistoryCalculator.h | 87 +++ include/llvm/CodeGen/DebugHandlerBase.h | 138 +++++ include/llvm/CodeGen/DwarfStringPoolEntry.h | 47 +- include/llvm/CodeGen/FunctionLoweringInfo.h | 3 +- include/llvm/CodeGen/GCMetadata.h | 9 +- include/llvm/CodeGen/GCMetadataPrinter.h | 6 + include/llvm/CodeGen/GCStrategy.h | 50 +- include/llvm/CodeGen/GCs.h | 46 -- include/llvm/CodeGen/GlobalISel/CSEInfo.h | 237 ++++++++ include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h | 110 ++++ include/llvm/CodeGen/GlobalISel/CallLowering.h | 10 +- include/llvm/CodeGen/GlobalISel/Combiner.h | 8 +- include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 19 +- include/llvm/CodeGen/GlobalISel/CombinerInfo.h | 16 +- .../CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h | 104 +--- .../llvm/CodeGen/GlobalISel/GISelChangeObserver.h | 111 ++++ include/llvm/CodeGen/GlobalISel/GISelWorkList.h | 35 +- include/llvm/CodeGen/GlobalISel/IRTranslator.h | 13 +- .../GlobalISel/LegalizationArtifactCombiner.h | 145 +++-- include/llvm/CodeGen/GlobalISel/LegalizerHelper.h | 12 +- include/llvm/CodeGen/GlobalISel/LegalizerInfo.h | 25 +- include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h | 471 +++++++++------- include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h | 11 +- include/llvm/CodeGen/GlobalISel/Utils.h | 3 + include/llvm/CodeGen/ISDOpcodes.h | 100 +++- include/llvm/CodeGen/LinkAllAsmWriterComponents.h | 2 +- include/llvm/CodeGen/LinkAllCodegenComponents.h | 8 +- include/llvm/CodeGen/LiveIntervals.h | 6 +- include/llvm/CodeGen/LivePhysRegs.h | 17 +- include/llvm/CodeGen/LiveRegUnits.h | 8 +- include/llvm/CodeGen/MIRYamlMapping.h | 7 + include/llvm/CodeGen/MachineBasicBlock.h | 6 + include/llvm/CodeGen/MachineFrameInfo.h | 42 +- include/llvm/CodeGen/MachineFunction.h | 119 ++-- include/llvm/CodeGen/MachineInstr.h | 281 ++++++++-- include/llvm/CodeGen/MachineInstrBuilder.h | 17 +- include/llvm/CodeGen/MachineModuleInfo.h | 8 +- include/llvm/CodeGen/MachineModuleInfoImpls.h | 22 + include/llvm/CodeGen/MachineOutliner.h | 78 ++- include/llvm/CodeGen/MachinePassRegistry.h | 80 ++- include/llvm/CodeGen/MachinePipeliner.h | 608 +++++++++++++++++++++ include/llvm/CodeGen/MachineRegisterInfo.h | 19 +- include/llvm/CodeGen/MachineScheduler.h | 25 +- include/llvm/CodeGen/Passes.h | 12 +- include/llvm/CodeGen/PreISelIntrinsicLowering.h | 3 +- include/llvm/CodeGen/PseudoSourceValue.h | 10 +- include/llvm/CodeGen/RegAllocRegistry.h | 16 +- include/llvm/CodeGen/RegisterUsageInfo.h | 6 +- include/llvm/CodeGen/ScheduleDAG.h | 23 +- include/llvm/CodeGen/ScheduleDAGInstrs.h | 3 +- include/llvm/CodeGen/SchedulerRegistry.h | 13 +- include/llvm/CodeGen/SelectionDAG.h | 119 ++-- include/llvm/CodeGen/SelectionDAGAddressAnalysis.h | 11 +- include/llvm/CodeGen/SelectionDAGISel.h | 12 + include/llvm/CodeGen/SelectionDAGNodes.h | 141 ++++- include/llvm/CodeGen/SlotIndexes.h | 16 +- include/llvm/CodeGen/StackMaps.h | 44 +- include/llvm/CodeGen/TargetFrameLowering.h | 7 +- include/llvm/CodeGen/TargetInstrInfo.h | 67 ++- include/llvm/CodeGen/TargetLowering.h | 304 +++++++++-- .../llvm/CodeGen/TargetLoweringObjectFileImpl.h | 2 + include/llvm/CodeGen/TargetPassConfig.h | 34 +- include/llvm/CodeGen/TargetRegisterInfo.h | 14 +- include/llvm/CodeGen/TargetSubtargetInfo.h | 38 ++ include/llvm/CodeGen/WasmEHFuncInfo.h | 4 +- 70 files changed, 3464 insertions(+), 1008 deletions(-) create mode 100644 include/llvm/CodeGen/AsmPrinterHandler.h create mode 100644 include/llvm/CodeGen/BuiltinGCs.h create mode 100644 include/llvm/CodeGen/DbgEntityHistoryCalculator.h create mode 100644 include/llvm/CodeGen/DebugHandlerBase.h delete mode 100644 include/llvm/CodeGen/GCs.h create mode 100644 include/llvm/CodeGen/GlobalISel/CSEInfo.h create mode 100644 include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h create mode 100644 include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h create mode 100644 include/llvm/CodeGen/MachinePipeliner.h (limited to 'include/llvm/CodeGen') diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index b6056380916c..413901d218f9 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -71,6 +71,7 @@ class MCTargetOptions; class MDNode; class Module; class raw_ostream; +class StackMaps; class TargetLoweringObjectFile; class TargetMachine; @@ -137,6 +138,9 @@ private: static char ID; +protected: + /// Protected struct HandlerInfo and Handlers permit target extended + /// AsmPrinter adds their own handlers. struct HandlerInfo { AsmPrinterHandler *Handler; const char *TimerName; @@ -365,6 +369,9 @@ public: /// emit the proxies we previously omitted in EmitGlobalVariable. void emitGlobalGOTEquivs(); + /// Emit the stack maps. + void emitStackMaps(StackMaps &SM); + //===------------------------------------------------------------------===// // Overridable Hooks //===------------------------------------------------------------------===// @@ -542,7 +549,7 @@ public: /// /// \p Value - The value to emit. /// \p Size - The size of the integer (in bytes) to emit. - virtual void EmitDebugThreadLocal(const MCExpr *Value, unsigned Size) const; + virtual void EmitDebugValue(const MCExpr *Value, unsigned Size) const; //===------------------------------------------------------------------===// // Dwarf Lowering Routines @@ -631,6 +638,11 @@ private: /// inline asm. void EmitInlineAsm(const MachineInstr *MI) const; + /// Add inline assembly info to the diagnostics machinery, so we can + /// emit file and position info. Returns SrcMgr memory buffer position. + unsigned addInlineAsmDiagBuffer(StringRef AsmStr, + const MDNode *LocMDNode) const; + //===------------------------------------------------------------------===// // Internal Implementation Details //===------------------------------------------------------------------===// @@ -647,6 +659,8 @@ private: void EmitLLVMUsedList(const ConstantArray *InitList); /// Emit llvm.ident metadata in an '.ident' directive. void EmitModuleIdents(Module &M); + /// Emit bytes for llvm.commandline metadata. + void EmitModuleCommandLines(Module &M); void EmitXXStructorList(const DataLayout &DL, const Constant *List, bool isCtor); diff --git a/include/llvm/CodeGen/AsmPrinterHandler.h b/include/llvm/CodeGen/AsmPrinterHandler.h new file mode 100644 index 000000000000..a8b13200dd4e --- /dev/null +++ b/include/llvm/CodeGen/AsmPrinterHandler.h @@ -0,0 +1,74 @@ +//===-- llvm/CodeGen/AsmPrinterHandler.h -----------------------*- C++ -*--===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains a generic interface for AsmPrinter handlers, +// like debug and EH info emitters. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_ASMPRINTERHANDLER_H +#define LLVM_CODEGEN_ASMPRINTERHANDLER_H + +#include "llvm/Support/DataTypes.h" + +namespace llvm { + +class AsmPrinter; +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class MCSymbol; + +typedef MCSymbol *ExceptionSymbolProvider(AsmPrinter *Asm); + +/// Collects and handles AsmPrinter objects required to build debug +/// or EH information. +class AsmPrinterHandler { +public: + virtual ~AsmPrinterHandler(); + + /// For symbols that have a size designated (e.g. common symbols), + /// this tracks that size. + virtual void setSymbolSize(const MCSymbol *Sym, uint64_t Size) = 0; + + /// Emit all sections that should come after the content. + virtual void endModule() = 0; + + /// Gather pre-function debug information. + /// Every beginFunction(MF) call should be followed by an endFunction(MF) + /// call. + virtual void beginFunction(const MachineFunction *MF) = 0; + + // Emit any of function marker (like .cfi_endproc). This is called + // before endFunction and cannot switch sections. + virtual void markFunctionEnd(); + + /// Gather post-function debug information. + /// Please note that some AsmPrinter implementations may not call + /// beginFunction at all. + virtual void endFunction(const MachineFunction *MF) = 0; + + virtual void beginFragment(const MachineBasicBlock *MBB, + ExceptionSymbolProvider ESP) {} + virtual void endFragment() {} + + /// Emit target-specific EH funclet machinery. + virtual void beginFunclet(const MachineBasicBlock &MBB, + MCSymbol *Sym = nullptr) {} + virtual void endFunclet() {} + + /// Process beginning of an instruction. + virtual void beginInstruction(const MachineInstr *MI) = 0; + + /// Process end of an instruction. + virtual void endInstruction() = 0; +}; +} // End of namespace llvm + +#endif diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h index f76a2426377a..f105d887c397 100644 --- a/include/llvm/CodeGen/BasicTTIImpl.h +++ b/include/llvm/CodeGen/BasicTTIImpl.h @@ -80,6 +80,23 @@ private: using BaseT = TargetTransformInfoImplCRTPBase; using TTI = TargetTransformInfo; + /// Estimate a cost of Broadcast as an extract and sequence of insert + /// operations. + unsigned getBroadcastShuffleOverhead(Type *Ty) { + assert(Ty->isVectorTy() && "Can only shuffle vectors"); + unsigned Cost = 0; + // Broadcast cost is equal to the cost of extracting the zero'th element + // plus the cost of inserting it into every element of the result vector. + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, Ty, 0); + + for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, Ty, i); + } + return Cost; + } + /// Estimate a cost of shuffle as a sequence of extract and insert /// operations. unsigned getPermuteShuffleOverhead(Type *Ty) { @@ -101,6 +118,50 @@ private: return Cost; } + /// Estimate a cost of subvector extraction as a sequence of extract and + /// insert operations. + unsigned getExtractSubvectorOverhead(Type *Ty, int Index, Type *SubTy) { + assert(Ty && Ty->isVectorTy() && SubTy && SubTy->isVectorTy() && + "Can only extract subvectors from vectors"); + int NumSubElts = SubTy->getVectorNumElements(); + assert((Index + NumSubElts) <= (int)Ty->getVectorNumElements() && + "SK_ExtractSubvector index out of range"); + + unsigned Cost = 0; + // Subvector extraction cost is equal to the cost of extracting element from + // the source type plus the cost of inserting them into the result vector + // type. + for (int i = 0; i != NumSubElts; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, Ty, i + Index); + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, SubTy, i); + } + return Cost; + } + + /// Estimate a cost of subvector insertion as a sequence of extract and + /// insert operations. + unsigned getInsertSubvectorOverhead(Type *Ty, int Index, Type *SubTy) { + assert(Ty && Ty->isVectorTy() && SubTy && SubTy->isVectorTy() && + "Can only insert subvectors into vectors"); + int NumSubElts = SubTy->getVectorNumElements(); + assert((Index + NumSubElts) <= (int)Ty->getVectorNumElements() && + "SK_InsertSubvector index out of range"); + + unsigned Cost = 0; + // Subvector insertion cost is equal to the cost of extracting element from + // the source type plus the cost of inserting them into the result vector + // type. + for (int i = 0; i != NumSubElts; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, SubTy, i); + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, Ty, i + Index); + } + return Cost; + } + /// Local query method delegates up to T which *must* implement this! const TargetSubtargetInfo *getST() const { return static_cast(this)->getST(); @@ -554,14 +615,20 @@ public: unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp) { switch (Kind) { + case TTI::SK_Broadcast: + return getBroadcastShuffleOverhead(Tp); case TTI::SK_Select: + case TTI::SK_Reverse: case TTI::SK_Transpose: case TTI::SK_PermuteSingleSrc: case TTI::SK_PermuteTwoSrc: return getPermuteShuffleOverhead(Tp); - default: - return 1; + case TTI::SK_ExtractSubvector: + return getExtractSubvectorOverhead(Tp, Index, SubTp); + case TTI::SK_InsertSubvector: + return getInsertSubvectorOverhead(Tp, Index, SubTp); } + llvm_unreachable("Unknown TTI::ShuffleKind"); } unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, @@ -783,8 +850,9 @@ public: unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, - unsigned Alignment, - unsigned AddressSpace) { + unsigned Alignment, unsigned AddressSpace, + bool UseMaskForCond = false, + bool UseMaskForGaps = false) { VectorType *VT = dyn_cast(VecTy); assert(VT && "Expect a vector type for interleaved memory op"); @@ -795,8 +863,13 @@ public: VectorType *SubVT = VectorType::get(VT->getElementType(), NumSubElts); // Firstly, the cost of load/store operation. - unsigned Cost = static_cast(this)->getMemoryOpCost( - Opcode, VecTy, Alignment, AddressSpace); + unsigned Cost; + if (UseMaskForCond || UseMaskForGaps) + Cost = static_cast(this)->getMaskedMemoryOpCost( + Opcode, VecTy, Alignment, AddressSpace); + else + Cost = static_cast(this)->getMemoryOpCost(Opcode, VecTy, Alignment, + AddressSpace); // Legalize the vector type, and get the legalized and unlegalized type // sizes. @@ -892,6 +965,40 @@ public: ->getVectorInstrCost(Instruction::InsertElement, VT, i); } + if (!UseMaskForCond) + return Cost; + + Type *I8Type = Type::getInt8Ty(VT->getContext()); + VectorType *MaskVT = VectorType::get(I8Type, NumElts); + SubVT = VectorType::get(I8Type, NumSubElts); + + // The Mask shuffling cost is extract all the elements of the Mask + // and insert each of them Factor times into the wide vector: + // + // E.g. an interleaved group with factor 3: + // %mask = icmp ult <8 x i32> %vec1, %vec2 + // %interleaved.mask = shufflevector <8 x i1> %mask, <8 x i1> undef, + // <24 x i32> <0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7> + // The cost is estimated as extract all mask elements from the <8xi1> mask + // vector and insert them factor times into the <24xi1> shuffled mask + // vector. + for (unsigned i = 0; i < NumSubElts; i++) + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, SubVT, i); + + for (unsigned i = 0; i < NumElts; i++) + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, MaskVT, i); + + // The Gaps mask is invariant and created outside the loop, therefore the + // cost of creating it is not accounted for here. However if we have both + // a MaskForGaps and some other mask that guards the execution of the + // memory access, we need to account for the cost of And-ing the two masks + // inside the loop. + if (UseMaskForGaps) + Cost += static_cast(this)->getArithmeticInstrCost( + BinaryOperator::And, MaskVT); + return Cost; } @@ -901,6 +1008,7 @@ public: unsigned VF = 1) { unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1); assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type"); + auto *ConcreteTTI = static_cast(this); switch (IID) { default: { @@ -926,29 +1034,24 @@ public: ScalarizationCost += getOperandsScalarizationOverhead(Args, VF); } - return static_cast(this)-> - getIntrinsicInstrCost(IID, RetTy, Types, FMF, ScalarizationCost); + return ConcreteTTI->getIntrinsicInstrCost(IID, RetTy, Types, FMF, + ScalarizationCost); } case Intrinsic::masked_scatter: { assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[3]; bool VarMask = !isa(Mask); unsigned Alignment = cast(Args[2])->getZExtValue(); - return - static_cast(this)->getGatherScatterOpCost(Instruction::Store, - Args[0]->getType(), - Args[1], VarMask, - Alignment); + return ConcreteTTI->getGatherScatterOpCost( + Instruction::Store, Args[0]->getType(), Args[1], VarMask, Alignment); } case Intrinsic::masked_gather: { assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[2]; bool VarMask = !isa(Mask); unsigned Alignment = cast(Args[1])->getZExtValue(); - return - static_cast(this)->getGatherScatterOpCost(Instruction::Load, - RetTy, Args[0], VarMask, - Alignment); + return ConcreteTTI->getGatherScatterOpCost(Instruction::Load, RetTy, + Args[0], VarMask, Alignment); } case Intrinsic::experimental_vector_reduce_add: case Intrinsic::experimental_vector_reduce_mul: @@ -964,6 +1067,45 @@ public: case Intrinsic::experimental_vector_reduce_umax: case Intrinsic::experimental_vector_reduce_umin: return getIntrinsicInstrCost(IID, RetTy, Args[0]->getType(), FMF); + case Intrinsic::fshl: + case Intrinsic::fshr: { + Value *X = Args[0]; + Value *Y = Args[1]; + Value *Z = Args[2]; + TTI::OperandValueProperties OpPropsX, OpPropsY, OpPropsZ, OpPropsBW; + TTI::OperandValueKind OpKindX = TTI::getOperandInfo(X, OpPropsX); + TTI::OperandValueKind OpKindY = TTI::getOperandInfo(Y, OpPropsY); + TTI::OperandValueKind OpKindZ = TTI::getOperandInfo(Z, OpPropsZ); + TTI::OperandValueKind OpKindBW = TTI::OK_UniformConstantValue; + OpPropsBW = isPowerOf2_32(RetTy->getScalarSizeInBits()) ? TTI::OP_PowerOf2 + : TTI::OP_None; + // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) + // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) + unsigned Cost = 0; + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Or, RetTy); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Sub, RetTy); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Shl, RetTy, + OpKindX, OpKindZ, OpPropsX); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::LShr, RetTy, + OpKindY, OpKindZ, OpPropsY); + // Non-constant shift amounts requires a modulo. + if (OpKindZ != TTI::OK_UniformConstantValue && + OpKindZ != TTI::OK_NonUniformConstantValue) + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::URem, RetTy, + OpKindZ, OpKindBW, OpPropsZ, + OpPropsBW); + // For non-rotates (X != Y) we must add shift-by-zero handling costs. + if (X != Y) { + Type *CondTy = Type::getInt1Ty(RetTy->getContext()); + if (RetVF > 1) + CondTy = VectorType::get(CondTy, RetVF); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, + CondTy, nullptr); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy, + CondTy, nullptr); + } + return Cost; + } } } @@ -1036,15 +1178,18 @@ public: case Intrinsic::fabs: ISDs.push_back(ISD::FABS); break; + case Intrinsic::canonicalize: + ISDs.push_back(ISD::FCANONICALIZE); + break; case Intrinsic::minnum: ISDs.push_back(ISD::FMINNUM); if (FMF.noNaNs()) - ISDs.push_back(ISD::FMINNAN); + ISDs.push_back(ISD::FMINIMUM); break; case Intrinsic::maxnum: ISDs.push_back(ISD::FMAXNUM); if (FMF.noNaNs()) - ISDs.push_back(ISD::FMAXNAN); + ISDs.push_back(ISD::FMAXIMUM); break; case Intrinsic::copysign: ISDs.push_back(ISD::FCOPYSIGN); @@ -1136,7 +1281,8 @@ public: SmallVector CustomCost; for (unsigned ISD : ISDs) { if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { - if (IID == Intrinsic::fabs && TLI->isFAbsFree(LT.second)) { + if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() && + TLI->isFAbsFree(LT.second)) { return 0; } @@ -1280,24 +1426,36 @@ public: LT.second.isVector() ? LT.second.getVectorNumElements() : 1; while (NumVecElts > MVTLen) { NumVecElts /= 2; + Type *SubTy = VectorType::get(ScalarTy, NumVecElts); // Assume the pairwise shuffles add a cost. ShuffleCost += (IsPairwise + 1) * ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); - ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, Ty); - Ty = VectorType::get(ScalarTy, NumVecElts); + NumVecElts, SubTy); + ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, SubTy); + Ty = SubTy; ++LongVectorCount; } + + NumReduxLevels -= LongVectorCount; + // The minimal length of the vector is limited by the real length of vector // operations performed on the current platform. That's why several final // reduction operations are performed on the vectors with the same // architecture-dependent length. - ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * - ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); - ArithCost += (NumReduxLevels - LongVectorCount) * + + // Non pairwise reductions need one shuffle per reduction level. Pairwise + // reductions need two shuffles on every level, but the last one. On that + // level one of the shuffles is <0, u, u, ...> which is identity. + unsigned NumShuffles = NumReduxLevels; + if (IsPairwise && NumReduxLevels >= 1) + NumShuffles += NumReduxLevels - 1; + ShuffleCost += NumShuffles * + ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, + 0, Ty); + ArithCost += NumReduxLevels * ConcreteTTI->getArithmeticInstrCost(Opcode, Ty); - return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true); + return ShuffleCost + ArithCost + + ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0); } /// Try to calculate op costs for min/max reduction operations. @@ -1327,37 +1485,46 @@ public: LT.second.isVector() ? LT.second.getVectorNumElements() : 1; while (NumVecElts > MVTLen) { NumVecElts /= 2; + Type *SubTy = VectorType::get(ScalarTy, NumVecElts); + CondTy = VectorType::get(ScalarCondTy, NumVecElts); + // Assume the pairwise shuffles add a cost. ShuffleCost += (IsPairwise + 1) * ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); + NumVecElts, SubTy); MinMaxCost += - ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + - ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, + ConcreteTTI->getCmpSelInstrCost(CmpOpcode, SubTy, CondTy, nullptr) + + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, SubTy, CondTy, nullptr); - Ty = VectorType::get(ScalarTy, NumVecElts); - CondTy = VectorType::get(ScalarCondTy, NumVecElts); + Ty = SubTy; ++LongVectorCount; } + + NumReduxLevels -= LongVectorCount; + // The minimal length of the vector is limited by the real length of vector // operations performed on the current platform. That's why several final // reduction opertions are perfomed on the vectors with the same // architecture-dependent length. - ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * - ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); + + // Non pairwise reductions need one shuffle per reduction level. Pairwise + // reductions need two shuffles on every level, but the last one. On that + // level one of the shuffles is <0, u, u, ...> which is identity. + unsigned NumShuffles = NumReduxLevels; + if (IsPairwise && NumReduxLevels >= 1) + NumShuffles += NumReduxLevels - 1; + ShuffleCost += NumShuffles * + ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, + 0, Ty); MinMaxCost += - (NumReduxLevels - LongVectorCount) * + NumReduxLevels * (ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, nullptr)); - // Need 3 extractelement instructions for scalarization + an additional - // scalar select instruction. + // The last min/max should be in vector registers and we counted it above. + // So just need a single extractelement. return ShuffleCost + MinMaxCost + - 3 * getScalarizationOverhead(Ty, /*Insert=*/false, - /*Extract=*/true) + - ConcreteTTI->getCmpSelInstrCost(Instruction::Select, ScalarTy, - ScalarCondTy, nullptr); + ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0); } unsigned getVectorSplitCost() { return 1; } diff --git a/include/llvm/CodeGen/BuiltinGCs.h b/include/llvm/CodeGen/BuiltinGCs.h new file mode 100644 index 000000000000..1767922fb5ac --- /dev/null +++ b/include/llvm/CodeGen/BuiltinGCs.h @@ -0,0 +1,33 @@ +//===-- BuiltinGCs.h - Garbage collector linkage hacks --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains hack functions to force linking in the builtin GC +// components. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GCS_H +#define LLVM_CODEGEN_GCS_H + +namespace llvm { + +/// FIXME: Collector instances are not useful on their own. These no longer +/// serve any purpose except to link in the plugins. + +/// Ensure the definition of the builtin GCs gets linked in +void linkAllBuiltinGCs(); + +/// Creates an ocaml-compatible metadata printer. +void linkOcamlGCPrinter(); + +/// Creates an erlang-compatible metadata printer. +void linkErlangGCPrinter(); +} + +#endif diff --git a/include/llvm/CodeGen/CommandFlags.inc b/include/llvm/CodeGen/CommandFlags.inc index 7d2d167289e0..568d329a5e8c 100644 --- a/include/llvm/CodeGen/CommandFlags.inc +++ b/include/llvm/CodeGen/CommandFlags.inc @@ -74,7 +74,8 @@ static cl::opt TMModel( static cl::opt CMModel( "code-model", cl::desc("Choose code model"), - cl::values(clEnumValN(CodeModel::Small, "small", "Small code model"), + cl::values(clEnumValN(CodeModel::Tiny, "tiny", "Tiny code model"), + clEnumValN(CodeModel::Small, "small", "Small code model"), clEnumValN(CodeModel::Kernel, "kernel", "Kernel code model"), clEnumValN(CodeModel::Medium, "medium", "Medium code model"), clEnumValN(CodeModel::Large, "large", "Large code model"))); @@ -113,10 +114,16 @@ static cl::opt FileType( clEnumValN(TargetMachine::CGFT_Null, "null", "Emit nothing, for performance testing"))); -static cl::opt - DisableFPElim("disable-fp-elim", - cl::desc("Disable frame pointer elimination optimization"), - cl::init(false)); +static cl::opt FramePointerUsage( + "frame-pointer", cl::desc("Specify frame pointer elimination optimization"), + cl::init(llvm::FramePointer::None), + cl::values( + clEnumValN(llvm::FramePointer::All, "all", + "Disable frame pointer elimination"), + clEnumValN(llvm::FramePointer::NonLeaf, "non-leaf", + "Disable frame pointer elimination for non-leaf frame"), + clEnumValN(llvm::FramePointer::None, "none", + "Enable frame pointer elimination"))); static cl::opt EnableUnsafeFPMath( "enable-unsafe-fp-math", @@ -367,9 +374,14 @@ setFunctionAttributes(StringRef CPU, StringRef Features, Module &M) { NewAttrs.addAttribute("target-cpu", CPU); if (!Features.empty()) NewAttrs.addAttribute("target-features", Features); - if (DisableFPElim.getNumOccurrences() > 0) - NewAttrs.addAttribute("no-frame-pointer-elim", - DisableFPElim ? "true" : "false"); + if (FramePointerUsage.getNumOccurrences() > 0) { + if (FramePointerUsage == llvm::FramePointer::All) + NewAttrs.addAttribute("frame-pointer", "all"); + else if (FramePointerUsage == llvm::FramePointer::NonLeaf) + NewAttrs.addAttribute("frame-pointer", "non-leaf"); + else if (FramePointerUsage == llvm::FramePointer::None) + NewAttrs.addAttribute("frame-pointer", "none"); + } if (DisableTailCalls.getNumOccurrences() > 0) NewAttrs.addAttribute("disable-tail-calls", toStringRef(DisableTailCalls)); diff --git a/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h new file mode 100644 index 000000000000..befc28f084e7 --- /dev/null +++ b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h @@ -0,0 +1,87 @@ +//===- llvm/CodeGen/DbgEntityHistoryCalculator.h ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H +#define LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include + +namespace llvm { + +class DILocalVariable; +class MachineFunction; +class MachineInstr; +class TargetRegisterInfo; + +// For each user variable, keep a list of instruction ranges where this variable +// is accessible. The variables are listed in order of appearance. +class DbgValueHistoryMap { + // Each instruction range starts with a DBG_VALUE instruction, specifying the + // location of a variable, which is assumed to be valid until the end of the + // range. If end is not specified, location is valid until the start + // instruction of the next instruction range, or until the end of the + // function. +public: + using InstrRange = std::pair; + using InstrRanges = SmallVector; + using InlinedEntity = std::pair; + using InstrRangesMap = MapVector; + +private: + InstrRangesMap VarInstrRanges; + +public: + void startInstrRange(InlinedEntity Var, const MachineInstr &MI); + void endInstrRange(InlinedEntity Var, const MachineInstr &MI); + + // Returns register currently describing @Var. If @Var is currently + // unaccessible or is not described by a register, returns 0. + unsigned getRegisterForVar(InlinedEntity Var) const; + + bool empty() const { return VarInstrRanges.empty(); } + void clear() { VarInstrRanges.clear(); } + InstrRangesMap::const_iterator begin() const { return VarInstrRanges.begin(); } + InstrRangesMap::const_iterator end() const { return VarInstrRanges.end(); } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif +}; + +/// For each inlined instance of a source-level label, keep the corresponding +/// DBG_LABEL instruction. The DBG_LABEL instruction could be used to generate +/// a temporary (assembler) label before it. +class DbgLabelInstrMap { +public: + using InlinedEntity = std::pair; + using InstrMap = MapVector; + +private: + InstrMap LabelInstr; + +public: + void addInstr(InlinedEntity Label, const MachineInstr &MI); + + bool empty() const { return LabelInstr.empty(); } + void clear() { LabelInstr.clear(); } + InstrMap::const_iterator begin() const { return LabelInstr.begin(); } + InstrMap::const_iterator end() const { return LabelInstr.end(); } +}; + +void calculateDbgEntityHistory(const MachineFunction *MF, + const TargetRegisterInfo *TRI, + DbgValueHistoryMap &DbgValues, + DbgLabelInstrMap &DbgLabels); + +} // end namespace llvm + +#endif // LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H diff --git a/include/llvm/CodeGen/DebugHandlerBase.h b/include/llvm/CodeGen/DebugHandlerBase.h new file mode 100644 index 000000000000..4f0d14d317f2 --- /dev/null +++ b/include/llvm/CodeGen/DebugHandlerBase.h @@ -0,0 +1,138 @@ +//===-- llvm/CodeGen/DebugHandlerBase.h -----------------------*- C++ -*--===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common functionality for different debug information format backends. +// LLVM currently supports DWARF and CodeView. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_DEBUGHANDLERBASE_H +#define LLVM_CODEGEN_DEBUGHANDLERBASE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/CodeGen/AsmPrinterHandler.h" +#include "llvm/CodeGen/DbgEntityHistoryCalculator.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/IR/DebugInfoMetadata.h" + +namespace llvm { + +class AsmPrinter; +class MachineInstr; +class MachineModuleInfo; + +/// Represents the location at which a variable is stored. +struct DbgVariableLocation { + /// Base register. + unsigned Register; + + /// Chain of offsetted loads necessary to load the value if it lives in + /// memory. Every load except for the last is pointer-sized. + SmallVector LoadChain; + + /// Present if the location is part of a larger variable. + llvm::Optional FragmentInfo; + + /// Extract a VariableLocation from a MachineInstr. + /// This will only work if Instruction is a debug value instruction + /// and the associated DIExpression is in one of the supported forms. + /// If these requirements are not met, the returned Optional will not + /// have a value. + static Optional + extractFromMachineInstruction(const MachineInstr &Instruction); +}; + +/// Base class for debug information backends. Common functionality related to +/// tracking which variables and scopes are alive at a given PC live here. +class DebugHandlerBase : public AsmPrinterHandler { +protected: + DebugHandlerBase(AsmPrinter *A); + + /// Target of debug info emission. + AsmPrinter *Asm; + + /// Collected machine module information. + MachineModuleInfo *MMI; + + /// Previous instruction's location information. This is used to + /// determine label location to indicate scope boundaries in debug info. + /// We track the previous instruction's source location (if not line 0), + /// whether it was a label, and its parent BB. + DebugLoc PrevInstLoc; + MCSymbol *PrevLabel = nullptr; + const MachineBasicBlock *PrevInstBB = nullptr; + + /// This location indicates end of function prologue and beginning of + /// function body. + DebugLoc PrologEndLoc; + + /// If nonnull, stores the current machine instruction we're processing. + const MachineInstr *CurMI = nullptr; + + LexicalScopes LScopes; + + /// History of DBG_VALUE and clobber instructions for each user + /// variable. Variables are listed in order of appearance. + DbgValueHistoryMap DbgValues; + + /// Mapping of inlined labels and DBG_LABEL machine instruction. + DbgLabelInstrMap DbgLabels; + + /// Maps instruction with label emitted before instruction. + /// FIXME: Make this private from DwarfDebug, we have the necessary accessors + /// for it. + DenseMap LabelsBeforeInsn; + + /// Maps instruction with label emitted after instruction. + DenseMap LabelsAfterInsn; + + /// Indentify instructions that are marking the beginning of or + /// ending of a scope. + void identifyScopeMarkers(); + + /// Ensure that a label will be emitted before MI. + void requestLabelBeforeInsn(const MachineInstr *MI) { + LabelsBeforeInsn.insert(std::make_pair(MI, nullptr)); + } + + /// Ensure that a label will be emitted after MI. + void requestLabelAfterInsn(const MachineInstr *MI) { + LabelsAfterInsn.insert(std::make_pair(MI, nullptr)); + } + + virtual void beginFunctionImpl(const MachineFunction *MF) = 0; + virtual void endFunctionImpl(const MachineFunction *MF) = 0; + virtual void skippedNonDebugFunction() {} + + // AsmPrinterHandler overrides. +public: + void beginInstruction(const MachineInstr *MI) override; + void endInstruction() override; + + void beginFunction(const MachineFunction *MF) override; + void endFunction(const MachineFunction *MF) override; + + /// Return Label preceding the instruction. + MCSymbol *getLabelBeforeInsn(const MachineInstr *MI); + + /// Return Label immediately following the instruction. + MCSymbol *getLabelAfterInsn(const MachineInstr *MI); + + /// Return the function-local offset of an instruction. A label for the + /// instruction \p MI should exist (\ref getLabelAfterInsn). + const MCExpr *getFunctionLocalOffsetAfterInsn(const MachineInstr *MI); + + /// If this type is derived from a base type then return base type size. + static uint64_t getBaseTypeSize(const DITypeRef TyRef); +}; + +} + +#endif diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h index e6c0483cfc35..8b1a7af17bbf 100644 --- a/include/llvm/CodeGen/DwarfStringPoolEntry.h +++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h @@ -10,6 +10,7 @@ #ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H #define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/StringMap.h" namespace llvm { @@ -18,34 +19,52 @@ class MCSymbol; /// Data for a string pool entry. struct DwarfStringPoolEntry { + static constexpr unsigned NotIndexed = -1; + MCSymbol *Symbol; unsigned Offset; unsigned Index; + + bool isIndexed() const { return Index != NotIndexed; } }; /// String pool entry reference. -struct DwarfStringPoolEntryRef { - const StringMapEntry *I = nullptr; +class DwarfStringPoolEntryRef { + PointerIntPair *, 1, bool> + MapEntryAndIndexed; + + const StringMapEntry *getMapEntry() const { + return MapEntryAndIndexed.getPointer(); + } public: DwarfStringPoolEntryRef() = default; - explicit DwarfStringPoolEntryRef( - const StringMapEntry &I) - : I(&I) {} + DwarfStringPoolEntryRef(const StringMapEntry &Entry, + bool Indexed) + : MapEntryAndIndexed(&Entry, Indexed) {} - explicit operator bool() const { return I; } + explicit operator bool() const { return getMapEntry(); } MCSymbol *getSymbol() const { - assert(I->second.Symbol && "No symbol available!"); - return I->second.Symbol; + assert(getMapEntry()->second.Symbol && "No symbol available!"); + return getMapEntry()->second.Symbol; } - unsigned getOffset() const { return I->second.Offset; } - unsigned getIndex() const { return I->second.Index; } - StringRef getString() const { return I->first(); } + unsigned getOffset() const { return getMapEntry()->second.Offset; } + bool isIndexed() const { return MapEntryAndIndexed.getInt(); } + unsigned getIndex() const { + assert(isIndexed()); + assert(getMapEntry()->getValue().isIndexed()); + return getMapEntry()->second.Index; + } + StringRef getString() const { return getMapEntry()->first(); } /// Return the entire string pool entry for convenience. - DwarfStringPoolEntry getEntry() const { return I->getValue(); } + DwarfStringPoolEntry getEntry() const { return getMapEntry()->getValue(); } - bool operator==(const DwarfStringPoolEntryRef &X) const { return I == X.I; } - bool operator!=(const DwarfStringPoolEntryRef &X) const { return I != X.I; } + bool operator==(const DwarfStringPoolEntryRef &X) const { + return getMapEntry() == X.getMapEntry(); + } + bool operator!=(const DwarfStringPoolEntryRef &X) const { + return getMapEntry() != X.getMapEntry(); + } }; } // end namespace llvm diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index 2da00b7d61ab..7c658515de09 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -1,4 +1,4 @@ -//===- FunctionLoweringInfo.h - Lower functions from LLVM IR to CodeGen ---===// +//===- FunctionLoweringInfo.h - Lower functions from LLVM IR ---*- C++ -*--===// // // The LLVM Compiler Infrastructure // @@ -246,6 +246,7 @@ public: return 0; unsigned &R = ValueMap[V]; assert(R == 0 && "Already initialized this value register!"); + assert(VirtReg2Value.empty()); return R = CreateRegs(V->getType()); } diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h index ad2599fc120e..7fb27202c122 100644 --- a/include/llvm/CodeGen/GCMetadata.h +++ b/include/llvm/CodeGen/GCMetadata.h @@ -55,12 +55,11 @@ class MCSymbol; /// GCPoint - Metadata for a collector-safe point in machine code. /// struct GCPoint { - GC::PointKind Kind; ///< The kind of the safe point. MCSymbol *Label; ///< A label. DebugLoc Loc; - GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL) - : Kind(K), Label(L), Loc(std::move(DL)) {} + GCPoint(MCSymbol *L, DebugLoc DL) + : Label(L), Loc(std::move(DL)) {} }; /// GCRoot - Metadata for a pointer to an object managed by the garbage @@ -124,8 +123,8 @@ public: /// addSafePoint - Notes the existence of a safe point. Num is the ID of the /// label just prior to the safe point (if the code generator is using /// MachineModuleInfo). - void addSafePoint(GC::PointKind Kind, MCSymbol *Label, const DebugLoc &DL) { - SafePoints.emplace_back(Kind, Label, DL); + void addSafePoint(MCSymbol *Label, const DebugLoc &DL) { + SafePoints.emplace_back(Label, DL); } /// getFrameSize/setFrameSize - Records the function's frame size. diff --git a/include/llvm/CodeGen/GCMetadataPrinter.h b/include/llvm/CodeGen/GCMetadataPrinter.h index 1cc69a7b71af..5f1efb2ce02c 100644 --- a/include/llvm/CodeGen/GCMetadataPrinter.h +++ b/include/llvm/CodeGen/GCMetadataPrinter.h @@ -29,6 +29,7 @@ class GCMetadataPrinter; class GCModuleInfo; class GCStrategy; class Module; +class StackMaps; /// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the /// defaults from Registry. @@ -60,6 +61,11 @@ public: /// Called after the assembly for the module is generated by /// the AsmPrinter (but before target specific hooks) virtual void finishAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {} + + /// Called when the stack maps are generated. Return true if + /// stack maps with a custom format are generated. Otherwise + /// returns false and the default format will be used. + virtual bool emitStackMaps(StackMaps &SM, AsmPrinter &AP) { return false; } }; } // end namespace llvm diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h index f835bacfb548..5a60cd7cb823 100644 --- a/include/llvm/CodeGen/GCStrategy.h +++ b/include/llvm/CodeGen/GCStrategy.h @@ -59,19 +59,6 @@ namespace llvm { class Type; -namespace GC { - -/// PointKind - Used to indicate whether the address of the call instruction -/// or the address after the call instruction is listed in the stackmap. For -/// most runtimes, PostCall safepoints are appropriate. -/// -enum PointKind { - PreCall, ///< Instr is a call instruction. - PostCall ///< Instr is the return address of a call. -}; - -} // end namespace GC - /// GCStrategy describes a garbage collector algorithm's code generation /// requirements, and provides overridable hooks for those needs which cannot /// be abstractly described. GCStrategy objects must be looked up through @@ -88,11 +75,7 @@ protected: /// if set, none of the other options can be /// anything but their default values. - unsigned NeededSafePoints = 0; ///< Bitmask of required safe points. - bool CustomReadBarriers = false; ///< Default is to insert loads. - bool CustomWriteBarriers = false; ///< Default is to insert stores. - bool CustomRoots = false; ///< Default is to pass through to backend. - bool InitRoots= true; ///< If set, roots are nulled during lowering. + bool NeededSafePoints = false; ///< if set, calls are inferred to be safepoints bool UsesMetadata = false; ///< If set, backend must emit metadata tables. public: @@ -103,16 +86,6 @@ public: /// name string specified on functions which use this strategy. const std::string &getName() const { return Name; } - /// By default, write barriers are replaced with simple store - /// instructions. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcwrite. - bool customWriteBarrier() const { return CustomWriteBarriers; } - - /// By default, read barriers are replaced with simple load - /// instructions. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcread. - bool customReadBarrier() const { return CustomReadBarriers; } - /// Returns true if this strategy is expecting the use of gc.statepoints, /// and false otherwise. bool useStatepoints() const { return UseStatepoints; } @@ -135,25 +108,8 @@ public: */ ///@{ - /// True if safe points of any kind are required. By default, none are - /// recorded. - bool needsSafePoints() const { return NeededSafePoints != 0; } - - /// True if the given kind of safe point is required. By default, none are - /// recorded. - bool needsSafePoint(GC::PointKind Kind) const { - return (NeededSafePoints & 1 << Kind) != 0; - } - - /// By default, roots are left for the code generator so it can generate a - /// stack map. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcroot. - bool customRoots() const { return CustomRoots; } - - /// If set, gcroot intrinsics should initialize their allocas to null - /// before the first use. This is necessary for most GCs and is enabled by - /// default. - bool initializeRoots() const { return InitRoots; } + /// True if safe points need to be inferred on call sites + bool needsSafePoints() const { return NeededSafePoints; } /// If set, appropriate metadata tables must be emitted by the back-end /// (assembler, JIT, or otherwise). For statepoint, this method is diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h deleted file mode 100644 index 5207f801c84e..000000000000 --- a/include/llvm/CodeGen/GCs.h +++ /dev/null @@ -1,46 +0,0 @@ -//===-- GCs.h - Garbage collector linkage hacks ---------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains hack functions to force linking in the GC components. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_GCS_H -#define LLVM_CODEGEN_GCS_H - -namespace llvm { -class GCStrategy; -class GCMetadataPrinter; - -/// FIXME: Collector instances are not useful on their own. These no longer -/// serve any purpose except to link in the plugins. - -/// Creates a CoreCLR-compatible garbage collector. -void linkCoreCLRGC(); - -/// Creates an ocaml-compatible garbage collector. -void linkOcamlGC(); - -/// Creates an ocaml-compatible metadata printer. -void linkOcamlGCPrinter(); - -/// Creates an erlang-compatible garbage collector. -void linkErlangGC(); - -/// Creates an erlang-compatible metadata printer. -void linkErlangGCPrinter(); - -/// Creates a shadow stack garbage collector. This collector requires no code -/// generator support. -void linkShadowStackGC(); - -void linkStatepointExampleGC(); -} - -#endif diff --git a/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/include/llvm/CodeGen/GlobalISel/CSEInfo.h new file mode 100644 index 000000000000..ce2d285a99e5 --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -0,0 +1,237 @@ +//===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Provides analysis for continuously CSEing during GISel passes. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEINFO_H +#define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H + +#include "llvm/ADT/FoldingSet.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GISelWorkList.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" +#include "llvm/Support/Allocator.h" + +namespace llvm { + +/// A class that wraps MachineInstrs and derives from FoldingSetNode in order to +/// be uniqued in a CSEMap. The tradeoff here is extra memory allocations for +/// UniqueMachineInstr vs making MachineInstr bigger. +class UniqueMachineInstr : public FoldingSetNode { + friend class GISelCSEInfo; + const MachineInstr *MI; + explicit UniqueMachineInstr(const MachineInstr *MI) : MI(MI) {} + +public: + void Profile(FoldingSetNodeID &ID); +}; + +// Class representing some configuration that can be done during CSE analysis. +// Currently it only supports shouldCSE method that each pass can set. +class CSEConfig { +public: + virtual ~CSEConfig() = default; + // Hook for defining which Generic instructions should be CSEd. + // GISelCSEInfo currently only calls this hook when dealing with generic + // opcodes. + virtual bool shouldCSEOpc(unsigned Opc); +}; + +// TODO: Find a better place for this. +// Commonly used for O0 config. +class CSEConfigConstantOnly : public CSEConfig { +public: + virtual ~CSEConfigConstantOnly() = default; + virtual bool shouldCSEOpc(unsigned Opc) override; +}; + +/// The CSE Analysis object. +/// This installs itself as a delegate to the MachineFunction to track +/// new instructions as well as deletions. It however will not be able to +/// track instruction mutations. In such cases, recordNewInstruction should be +/// called (for eg inside MachineIRBuilder::recordInsertion). +/// Also because of how just the instruction can be inserted without adding any +/// operands to the instruction, instructions are uniqued and inserted lazily. +/// CSEInfo should assert when trying to enter an incomplete instruction into +/// the CSEMap. There is Opcode level granularity on which instructions can be +/// CSE'd and for now, only Generic instructions are CSEable. +class GISelCSEInfo : public GISelChangeObserver { + // Make it accessible only to CSEMIRBuilder. + friend class CSEMIRBuilder; + + BumpPtrAllocator UniqueInstrAllocator; + FoldingSet CSEMap; + MachineRegisterInfo *MRI = nullptr; + MachineFunction *MF = nullptr; + std::unique_ptr CSEOpt; + /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel, + /// often instructions are mutated (while their ID has completely changed). + /// Whenever mutation happens, invalidate the UniqueMachineInstr for the + /// MachineInstr + DenseMap InstrMapping; + + /// Store instructions that are not fully formed in TemporaryInsts. + /// Also because CSE insertion happens lazily, we can remove insts from this + /// list and avoid inserting and then removing from the CSEMap. + GISelWorkList<8> TemporaryInsts; + + // Only used in asserts. + DenseMap OpcodeHitTable; + + bool isUniqueMachineInstValid(const UniqueMachineInstr &UMI) const; + + void invalidateUniqueMachineInstr(UniqueMachineInstr *UMI); + + UniqueMachineInstr *getNodeIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, void *&InsertPos); + + /// Allocate and construct a new UniqueMachineInstr for MI and return. + UniqueMachineInstr *getUniqueInstrForMI(const MachineInstr *MI); + + void insertNode(UniqueMachineInstr *UMI, void *InsertPos = nullptr); + + /// Get the MachineInstr(Unique) if it exists already in the CSEMap and the + /// same MachineBasicBlock. + MachineInstr *getMachineInstrIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, + void *&InsertPos); + + /// Use this method to allocate a new UniqueMachineInstr for MI and insert it + /// into the CSEMap. MI should return true for shouldCSE(MI->getOpcode()) + void insertInstr(MachineInstr *MI, void *InsertPos = nullptr); + +public: + GISelCSEInfo() = default; + + virtual ~GISelCSEInfo(); + + void setMF(MachineFunction &MF); + + /// Records a newly created inst in a list and lazily insert it to the CSEMap. + /// Sometimes, this method might be called with a partially constructed + /// MachineInstr, + // (right after BuildMI without adding any operands) - and in such cases, + // defer the hashing of the instruction to a later stage. + void recordNewInstruction(MachineInstr *MI); + + /// Use this callback to inform CSE about a newly fully created instruction. + void handleRecordedInst(MachineInstr *MI); + + /// Use this callback to insert all the recorded instructions. At this point, + /// all of these insts need to be fully constructed and should not be missing + /// any operands. + void handleRecordedInsts(); + + /// Remove this inst from the CSE map. If this inst has not been inserted yet, + /// it will be removed from the Tempinsts list if it exists. + void handleRemoveInst(MachineInstr *MI); + + void releaseMemory(); + + void setCSEConfig(std::unique_ptr Opt) { CSEOpt = std::move(Opt); } + + bool shouldCSE(unsigned Opc) const; + + void analyze(MachineFunction &MF); + + void countOpcodeHit(unsigned Opc); + + void print(); + + // Observer API + void erasingInstr(MachineInstr &MI) override; + void createdInstr(MachineInstr &MI) override; + void changingInstr(MachineInstr &MI) override; + void changedInstr(MachineInstr &MI) override; +}; + +class TargetRegisterClass; +class RegisterBank; + +// Simple builder class to easily profile properties about MIs. +class GISelInstProfileBuilder { + FoldingSetNodeID &ID; + const MachineRegisterInfo &MRI; + +public: + GISelInstProfileBuilder(FoldingSetNodeID &ID, const MachineRegisterInfo &MRI) + : ID(ID), MRI(MRI) {} + // Profiling methods. + const GISelInstProfileBuilder &addNodeIDOpcode(unsigned Opc) const; + const GISelInstProfileBuilder &addNodeIDRegType(const LLT &Ty) const; + const GISelInstProfileBuilder &addNodeIDRegType(const unsigned) const; + + const GISelInstProfileBuilder & + addNodeIDRegType(const TargetRegisterClass *RC) const; + const GISelInstProfileBuilder &addNodeIDRegType(const RegisterBank *RB) const; + + const GISelInstProfileBuilder &addNodeIDRegNum(unsigned Reg) const; + + const GISelInstProfileBuilder &addNodeIDImmediate(int64_t Imm) const; + const GISelInstProfileBuilder & + addNodeIDMBB(const MachineBasicBlock *MBB) const; + + const GISelInstProfileBuilder & + addNodeIDMachineOperand(const MachineOperand &MO) const; + + const GISelInstProfileBuilder &addNodeIDFlag(unsigned Flag) const; + const GISelInstProfileBuilder &addNodeID(const MachineInstr *MI) const; +}; + +/// Simple wrapper that does the following. +/// 1) Lazily evaluate the MachineFunction to compute CSEable instructions. +/// 2) Allows configuration of which instructions are CSEd through CSEConfig +/// object. Provides a method called get which takes a CSEConfig object. +class GISelCSEAnalysisWrapper { + GISelCSEInfo Info; + MachineFunction *MF = nullptr; + bool AlreadyComputed = false; + +public: + /// Takes a CSEConfig object that defines what opcodes get CSEd. + /// If CSEConfig is already set, and the CSE Analysis has been preserved, + /// it will not use the new CSEOpt(use Recompute to force using the new + /// CSEOpt). + GISelCSEInfo &get(std::unique_ptr CSEOpt, bool ReCompute = false); + void setMF(MachineFunction &MFunc) { MF = &MFunc; } + void setComputed(bool Computed) { AlreadyComputed = Computed; } + void releaseMemory() { Info.releaseMemory(); } +}; + +/// The actual analysis pass wrapper. +class GISelCSEAnalysisWrapperPass : public MachineFunctionPass { + GISelCSEAnalysisWrapper Wrapper; + +public: + static char ID; + GISelCSEAnalysisWrapperPass() : MachineFunctionPass(ID) { + initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + const GISelCSEAnalysisWrapper &getCSEWrapper() const { return Wrapper; } + GISelCSEAnalysisWrapper &getCSEWrapper() { return Wrapper; } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void releaseMemory() override { + Wrapper.releaseMemory(); + Wrapper.setComputed(false); + } +}; + +} // namespace llvm + +#endif diff --git a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h new file mode 100644 index 000000000000..a8fb736ebbb5 --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -0,0 +1,110 @@ +//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h --*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements a version of MachineIRBuilder which CSEs insts within +/// a MachineBasicBlock. +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H +#define LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H + +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" + +namespace llvm { + +/// Defines a builder that does CSE of MachineInstructions using GISelCSEInfo. +/// Eg usage. +/// +/// +/// GISelCSEInfo *Info = +/// &getAnalysis().getCSEInfo(); CSEMIRBuilder +/// CB(Builder.getState()); CB.setCSEInfo(Info); auto A = CB.buildConstant(s32, +/// 42); auto B = CB.buildConstant(s32, 42); assert(A == B); unsigned CReg = +/// MRI.createGenericVirtualRegister(s32); auto C = CB.buildConstant(CReg, 42); +/// assert(C->getOpcode() == TargetOpcode::COPY); +/// Explicitly passing in a register would materialize a copy if possible. +/// CSEMIRBuilder also does trivial constant folding for binary ops. +class CSEMIRBuilder : public MachineIRBuilder { + + /// Returns true if A dominates B (within the same basic block). + /// Both iterators must be in the same basic block. + // + // TODO: Another approach for checking dominance is having two iterators and + // making them go towards each other until they meet or reach begin/end. Which + // approach is better? Should this even change dynamically? For G_CONSTANTS + // most of which will be at the top of the BB, the top down approach would be + // a better choice. Does IRTranslator placing constants at the beginning still + // make sense? Should this change based on Opcode? + bool dominates(MachineBasicBlock::const_iterator A, + MachineBasicBlock::const_iterator B) const; + + /// For given ID, find a machineinstr in the CSE Map. If found, check if it + /// dominates the current insertion point and if not, move it just before the + /// current insertion point and return it. If not found, return Null + /// MachineInstrBuilder. + MachineInstrBuilder getDominatingInstrForID(FoldingSetNodeID &ID, + void *&NodeInsertPos); + /// Simple check if we can CSE (we have the CSEInfo) or if this Opcode is + /// safe to CSE. + bool canPerformCSEForOpc(unsigned Opc) const; + + void profileDstOp(const DstOp &Op, GISelInstProfileBuilder &B) const; + + void profileDstOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const DstOp &Op : Ops) + profileDstOp(Op, B); + } + + void profileSrcOp(const SrcOp &Op, GISelInstProfileBuilder &B) const; + + void profileSrcOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const SrcOp &Op : Ops) + profileSrcOp(Op, B); + } + + void profileMBBOpcode(GISelInstProfileBuilder &B, unsigned Opc) const; + + void profileEverything(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, Optional Flags, + GISelInstProfileBuilder &B) const; + + // Takes a MachineInstrBuilder and inserts it into the CSEMap using the + // NodeInsertPos. + MachineInstrBuilder memoizeMI(MachineInstrBuilder MIB, void *NodeInsertPos); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // we emit a copy from the CSE'd inst to the VReg. + MachineInstrBuilder generateCopiesIfRequired(ArrayRef DstOps, + MachineInstrBuilder &MIB); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // check if we can generate copies. It's not possible to return a single MIB, + // while emitting copies to multiple vregs. + bool checkCopyToDefsPossible(ArrayRef DstOps); + +public: + // Pull in base class constructors. + using MachineIRBuilder::MachineIRBuilder; + // Unhide buildInstr + MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flag = None) override; + // Bring in the other overload from the base class. + using MachineIRBuilder::buildConstant; + + MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val) override; + + // Bring in the other overload from the base class. + using MachineIRBuilder::buildFConstant; + MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val) override; +}; +} // namespace llvm +#endif diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h index 58eb412d8c24..ab498e8f070b 100644 --- a/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -40,6 +40,7 @@ class Value; class CallLowering { const TargetLowering *TLI; + virtual void anchor(); public: struct ArgInfo { unsigned Reg; @@ -108,6 +109,9 @@ public: MachineIRBuilder &MIRBuilder; MachineRegisterInfo &MRI; CCAssignFn *AssignFn; + + private: + virtual void anchor(); }; protected: @@ -138,12 +142,12 @@ public: virtual ~CallLowering() = default; /// This hook must be implemented to lower outgoing return values, described - /// by \p Val, into the specified virtual register \p VReg. + /// by \p Val, into the specified virtual registers \p VRegs. /// This hook is used by GlobalISel. /// /// \return True if the lowering succeeds, false otherwise. - virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, - const Value *Val, unsigned VReg) const { + virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef VRegs) const { return false; } diff --git a/include/llvm/CodeGen/GlobalISel/Combiner.h b/include/llvm/CodeGen/GlobalISel/Combiner.h index 36a33deb4a64..b097c7817762 100644 --- a/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -21,6 +21,7 @@ namespace llvm { class MachineRegisterInfo; class CombinerInfo; +class GISelCSEInfo; class TargetPassConfig; class MachineFunction; @@ -28,14 +29,17 @@ class Combiner { public: Combiner(CombinerInfo &CombinerInfo, const TargetPassConfig *TPC); - bool combineMachineInstrs(MachineFunction &MF); + /// If CSEInfo is not null, then the Combiner will setup observer for + /// CSEInfo and instantiate a CSEMIRBuilder. Pass nullptr if CSE is not + /// needed. + bool combineMachineInstrs(MachineFunction &MF, GISelCSEInfo *CSEInfo); protected: CombinerInfo &CInfo; MachineRegisterInfo *MRI = nullptr; const TargetPassConfig *TPC; - MachineIRBuilder Builder; + std::unique_ptr Builder; }; } // End namespace llvm. diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 5d5b8398452c..6e9ac01c1ee2 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -1,4 +1,4 @@ -//== llvm/CodeGen/GlobalISel/CombinerHelper.h -------------- -*- C++ -*-==// +//===-- llvm/CodeGen/GlobalISel/CombinerHelper.h --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -20,21 +20,36 @@ namespace llvm { +class GISelChangeObserver; class MachineIRBuilder; class MachineRegisterInfo; class MachineInstr; +class MachineOperand; class CombinerHelper { MachineIRBuilder &Builder; MachineRegisterInfo &MRI; + GISelChangeObserver &Observer; public: - CombinerHelper(MachineIRBuilder &B); + CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B); + + /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes + void replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, unsigned ToReg) const; + + /// Replace a single register operand with a new register and inform the + /// observer of the changes. + void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, + unsigned ToReg) const; /// If \p MI is COPY, try to combine it. /// Returns true if MI changed. bool tryCombineCopy(MachineInstr &MI); + /// If \p MI is extend that consumes the result of a load, try to combine it. + /// Returns true if MI changed. + bool tryCombineExtendingLoads(MachineInstr &MI); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index 1d248547adbf..d21aa3f725d9 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -17,10 +17,12 @@ #include namespace llvm { +class GISelChangeObserver; class LegalizerInfo; class MachineInstr; class MachineIRBuilder; class MachineRegisterInfo; + // Contains information relevant to enabling/disabling various combines for a // pass. class CombinerInfo { @@ -41,7 +43,19 @@ public: /// illegal ops that are created. bool LegalizeIllegalOps; // TODO: Make use of this. const LegalizerInfo *LInfo; - virtual bool combine(MachineInstr &MI, MachineIRBuilder &B) const = 0; + + /// Attempt to combine instructions using MI as the root. + /// + /// Use Observer to report the creation, modification, and erasure of + /// instructions. GISelChangeObserver will automatically report certain + /// kinds of operations. These operations are: + /// * Instructions that are newly inserted into the MachineFunction + /// * Instructions that are erased from the MachineFunction. + /// + /// However, it is important to report instruction modification and this is + /// not automatic. + virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI, + MachineIRBuilder &B) const = 0; }; } // namespace llvm diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h index 8d61f9a68279..220a571b21db 100644 --- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -15,91 +15,20 @@ namespace llvm { -static Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, - const unsigned Op2, - const MachineRegisterInfo &MRI) { - auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI); - auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI); - if (MaybeOp1Cst && MaybeOp2Cst) { - LLT Ty = MRI.getType(Op1); - APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true); - APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true); - switch (Opcode) { - default: - break; - case TargetOpcode::G_ADD: - return C1 + C2; - case TargetOpcode::G_AND: - return C1 & C2; - case TargetOpcode::G_ASHR: - return C1.ashr(C2); - case TargetOpcode::G_LSHR: - return C1.lshr(C2); - case TargetOpcode::G_MUL: - return C1 * C2; - case TargetOpcode::G_OR: - return C1 | C2; - case TargetOpcode::G_SHL: - return C1 << C2; - case TargetOpcode::G_SUB: - return C1 - C2; - case TargetOpcode::G_XOR: - return C1 ^ C2; - case TargetOpcode::G_UDIV: - if (!C2.getBoolValue()) - break; - return C1.udiv(C2); - case TargetOpcode::G_SDIV: - if (!C2.getBoolValue()) - break; - return C1.sdiv(C2); - case TargetOpcode::G_UREM: - if (!C2.getBoolValue()) - break; - return C1.urem(C2); - case TargetOpcode::G_SREM: - if (!C2.getBoolValue()) - break; - return C1.srem(C2); - } - } - return None; -} - /// An MIRBuilder which does trivial constant folding of binary ops. /// Calls to buildInstr will also try to constant fold binary ops. -class ConstantFoldingMIRBuilder - : public FoldableInstructionsBuilder { +class ConstantFoldingMIRBuilder : public MachineIRBuilder { public: // Pull in base class constructors. - using FoldableInstructionsBuilder< - ConstantFoldingMIRBuilder>::FoldableInstructionsBuilder; - // Unhide buildInstr - using FoldableInstructionsBuilder::buildInstr; + using MachineIRBuilder::MachineIRBuilder; - // Implement buildBinaryOp required by FoldableInstructionsBuilder which - // tries to constant fold. - MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, - unsigned Src0, unsigned Src1) { - validateBinaryOp(Dst, Src0, Src1); - auto MaybeCst = ConstantFoldBinOp(Opcode, Src0, Src1, getMF().getRegInfo()); - if (MaybeCst) - return buildConstant(Dst, MaybeCst->getSExtValue()); - return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); - } - - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, UseArg1Ty &&Arg1, - UseArg2Ty &&Arg2) { - unsigned Dst = getDestFromArg(Ty); - return buildInstr(Opc, Dst, getRegFromArg(std::forward(Arg1)), - getRegFromArg(std::forward(Arg2))); - } + virtual ~ConstantFoldingMIRBuilder() = default; // Try to provide an overload for buildInstr for binary ops in order to // constant fold. - MachineInstrBuilder buildInstr(unsigned Opc, unsigned Dst, unsigned Src0, - unsigned Src1) { + MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flags = None) override { switch (Opc) { default: break; @@ -116,19 +45,18 @@ public: case TargetOpcode::G_SDIV: case TargetOpcode::G_UREM: case TargetOpcode::G_SREM: { - return buildBinaryOp(Opc, Dst, Src0, Src1); + assert(DstOps.size() == 1 && "Invalid dst ops"); + assert(SrcOps.size() == 2 && "Invalid src ops"); + const DstOp &Dst = DstOps[0]; + const SrcOp &Src0 = SrcOps[0]; + const SrcOp &Src1 = SrcOps[1]; + if (auto MaybeCst = + ConstantFoldBinOp(Opc, Src0.getReg(), Src1.getReg(), *getMRI())) + return buildConstant(Dst, MaybeCst->getSExtValue()); + break; } } - return buildInstr(Opc).addDef(Dst).addUse(Src0).addUse(Src1); - } - - // Fallback implementation of buildInstr. - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, - UseArgsTy &&... Args) { - auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - addUsesFromArgs(MIB, std::forward(Args)...); - return MIB; + return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps); } }; } // namespace llvm diff --git a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h new file mode 100644 index 000000000000..c8e8a7a5a7cb --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -0,0 +1,111 @@ +//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// This contains common code to allow clients to notify changes to machine +/// instr. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H +#define LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H + +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/CodeGen/MachineFunction.h" + +namespace llvm { +class MachineInstr; +class MachineRegisterInfo; + +/// Abstract class that contains various methods for clients to notify about +/// changes. This should be the preferred way for APIs to notify changes. +/// Typically calling erasingInstr/createdInstr multiple times should not affect +/// the result. The observer would likely need to check if it was already +/// notified earlier (consider using GISelWorkList). +class GISelChangeObserver { + SmallPtrSet ChangingAllUsesOfReg; + +public: + virtual ~GISelChangeObserver() {} + + /// An instruction is about to be erased. + virtual void erasingInstr(MachineInstr &MI) = 0; + /// An instruction was created and inserted into the function. + virtual void createdInstr(MachineInstr &MI) = 0; + /// This instruction is about to be mutated in some way. + virtual void changingInstr(MachineInstr &MI) = 0; + /// This instruction was mutated in some way. + virtual void changedInstr(MachineInstr &MI) = 0; + + /// All the instructions using the given register are being changed. + /// For convenience, finishedChangingAllUsesOfReg() will report the completion + /// of the changes. The use list may change between this call and + /// finishedChangingAllUsesOfReg(). + void changingAllUsesOfReg(const MachineRegisterInfo &MRI, unsigned Reg); + /// All instructions reported as changing by changingAllUsesOfReg() have + /// finished being changed. + void finishedChangingAllUsesOfReg(); + +}; + +/// Simple wrapper observer that takes several observers, and calls +/// each one for each event. If there are multiple observers (say CSE, +/// Legalizer, Combiner), it's sufficient to register this to the machine +/// function as the delegate. +class GISelObserverWrapper : public MachineFunction::Delegate, + public GISelChangeObserver { + SmallVector Observers; + +public: + GISelObserverWrapper() = default; + GISelObserverWrapper(ArrayRef Obs) + : Observers(Obs.begin(), Obs.end()) {} + // Adds an observer. + void addObserver(GISelChangeObserver *O) { Observers.push_back(O); } + // Removes an observer from the list and does nothing if observer is not + // present. + void removeObserver(GISelChangeObserver *O) { + auto It = std::find(Observers.begin(), Observers.end(), O); + if (It != Observers.end()) + Observers.erase(It); + } + // API for Observer. + void erasingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->erasingInstr(MI); + } + void createdInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->createdInstr(MI); + } + void changingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changingInstr(MI); + } + void changedInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changedInstr(MI); + } + // API for MachineFunction::Delegate + void MF_HandleInsertion(MachineInstr &MI) override { createdInstr(MI); } + void MF_HandleRemoval(MachineInstr &MI) override { erasingInstr(MI); } +}; + +/// A simple RAII based CSEInfo installer. +/// Use this in a scope to install a delegate to the MachineFunction and reset +/// it at the end of the scope. +class RAIIDelegateInstaller { + MachineFunction &MF; + MachineFunction::Delegate *Delegate; + +public: + RAIIDelegateInstaller(MachineFunction &MF, MachineFunction::Delegate *Del); + ~RAIIDelegateInstaller(); +}; + +} // namespace llvm +#endif diff --git a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 167905dc9aa1..1571841a208d 100644 --- a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -12,38 +12,42 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineInstr.h" #include "llvm/Support/Debug.h" namespace llvm { class MachineInstr; +class MachineFunction; -// Worklist which mostly works similar to InstCombineWorkList, but on MachineInstrs. -// The main difference with something like a SetVector is that erasing an element doesn't -// move all elements over one place - instead just nulls out the element of the vector. -// FIXME: Does it make sense to factor out common code with the instcombinerWorkList? +// Worklist which mostly works similar to InstCombineWorkList, but on +// MachineInstrs. The main difference with something like a SetVector is that +// erasing an element doesn't move all elements over one place - instead just +// nulls out the element of the vector. +// +// FIXME: Does it make sense to factor out common code with the +// instcombinerWorkList? template class GISelWorkList { - SmallVector Worklist; - DenseMap WorklistMap; + SmallVector Worklist; + DenseMap WorklistMap; public: - GISelWorkList() = default; + GISelWorkList() {} bool empty() const { return WorklistMap.empty(); } unsigned size() const { return WorklistMap.size(); } - /// Add - Add the specified instruction to the worklist if it isn't already - /// in it. + /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { - if (WorklistMap.try_emplace(I, Worklist.size()).second) { + if (WorklistMap.try_emplace(I, Worklist.size()).second) Worklist.push_back(I); - } } - /// Remove - remove I from the worklist if it exists. - void remove(MachineInstr *I) { + /// Remove I from the worklist if it exists. + void remove(const MachineInstr *I) { auto It = WorklistMap.find(I); if (It == WorklistMap.end()) return; // Not in worklist. @@ -53,6 +57,11 @@ public: WorklistMap.erase(It); } + void clear() { + Worklist.clear(); + WorklistMap.clear(); + } + MachineInstr *pop_back_val() { MachineInstr *I; do { diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index f3553966fcdf..d1770bf6e4ce 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -21,11 +21,11 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/Support/Allocator.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Allocator.h" #include #include @@ -232,6 +232,7 @@ private: /// Returns true if the value should be split into multiple LLTs. /// If \p Offsets is given then the split type's offsets will be stored in it. + /// If \p Offsets is not empty it will be cleared first. bool valueIsSplit(const Value &V, SmallVectorImpl *Offsets = nullptr); @@ -299,6 +300,8 @@ private: bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder); + bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) { return translateBinaryOp(TargetOpcode::G_ADD, U, MIRBuilder); } @@ -441,11 +444,13 @@ private: // I.e., compared to regular MIBuilder, this one also inserts the instruction // in the current block, it can creates block, etc., basically a kind of // IRBuilder, but for Machine IR. - MachineIRBuilder CurBuilder; + // CSEMIRBuilder CurBuilder; + std::unique_ptr CurBuilder; // Builder set to the entry block (just after ABI lowering instructions). Used // as a convenient location for Constants. - MachineIRBuilder EntryBuilder; + // CSEMIRBuilder EntryBuilder; + std::unique_ptr EntryBuilder; // The MachineFunction currently being translated. MachineFunction *MF; diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 873587651efd..20bec7650179 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h --===========// +//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-// // // The LLVM Compiler Infrastructure // @@ -14,12 +14,14 @@ #include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "legalizer" +using namespace llvm::MIPatternMatch; namespace llvm { class LegalizationArtifactCombiner { @@ -36,15 +38,29 @@ public: SmallVectorImpl &DeadInsts) { if (MI.getOpcode() != TargetOpcode::G_ANYEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // aext(trunc x) - > aext/copy/trunc x + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = DefMI->getOperand(1).getReg(); - Builder.setInstr(MI); - // We get a copy/trunc/extend depending on the sizes - Builder.buildAnyExtOrTrunc(DstReg, SrcReg); - markInstAndDefDead(MI, *DefMI, DeadInsts); + Builder.buildAnyExtOrTrunc(DstReg, TruncSrc); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); + return true; + } + + // aext([asz]ext x) -> [asz]ext x + unsigned ExtSrc; + MachineInstr *ExtMI; + if (mi_match(SrcReg, MRI, + m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)), + m_GSExt(m_Reg(ExtSrc)), + m_GZExt(m_Reg(ExtSrc)))))) { + Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc}); + markInstAndDefDead(MI, *ExtMI, DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); @@ -55,24 +71,25 @@ public: if (MI.getOpcode() != TargetOpcode::G_ZEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { - unsigned DstReg = MI.getOperand(0).getReg(); + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // zext(trunc x) - > and (aext/copy/trunc x), mask + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) || isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - Builder.setInstr(MI); - unsigned ZExtSrc = MI.getOperand(1).getReg(); - LLT ZExtSrcTy = MRI.getType(ZExtSrc); - APInt Mask = APInt::getAllOnesValue(ZExtSrcTy.getSizeInBits()); - auto MaskCstMIB = Builder.buildConstant(DstTy, Mask.getZExtValue()); - unsigned TruncSrc = DefMI->getOperand(1).getReg(); - // We get a copy/trunc/extend depending on the sizes - auto SrcCopyOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc); - Builder.buildAnd(DstReg, SrcCopyOrTrunc, MaskCstMIB); - markInstAndDefDead(MI, *DefMI, DeadInsts); + LLT SrcTy = MRI.getType(SrcReg); + APInt Mask = APInt::getAllOnesValue(SrcTy.getSizeInBits()); + auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue()); + Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), + MIBMask); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); @@ -83,33 +100,34 @@ public: if (MI.getOpcode() != TargetOpcode::G_SEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { - unsigned DstReg = MI.getOperand(0).getReg(); + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) || isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) || isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - Builder.setInstr(MI); - unsigned SExtSrc = MI.getOperand(1).getReg(); - LLT SExtSrcTy = MRI.getType(SExtSrc); - unsigned SizeDiff = DstTy.getSizeInBits() - SExtSrcTy.getSizeInBits(); - auto SizeDiffMIB = Builder.buildConstant(DstTy, SizeDiff); - unsigned TruncSrcReg = DefMI->getOperand(1).getReg(); - // We get a copy/trunc/extend depending on the sizes - auto SrcCopyExtOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrcReg); - auto ShlMIB = Builder.buildInstr(TargetOpcode::G_SHL, DstTy, - SrcCopyExtOrTrunc, SizeDiffMIB); - Builder.buildInstr(TargetOpcode::G_ASHR, DstReg, ShlMIB, SizeDiffMIB); - markInstAndDefDead(MI, *DefMI, DeadInsts); + LLT SrcTy = MRI.getType(SrcReg); + unsigned ShAmt = DstTy.getSizeInBits() - SrcTy.getSizeInBits(); + auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt); + auto MIBShl = Builder.buildInstr( + TargetOpcode::G_SHL, {DstTy}, + {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBShAmt}); + Builder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {MIBShl, MIBShAmt}); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); } - /// Try to fold sb = EXTEND (G_IMPLICIT_DEF sa) -> sb = G_IMPLICIT_DEF + /// Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF). bool tryFoldImplicitDef(MachineInstr &MI, SmallVectorImpl &DeadInsts) { unsigned Opcode = MI.getOpcode(); @@ -119,13 +137,25 @@ public: if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(), MRI)) { + Builder.setInstr(MI); unsigned DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) - return false; - LLVM_DEBUG(dbgs() << ".. Combine EXT(IMPLICIT_DEF) " << MI;); - Builder.setInstr(MI); - Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, DstReg); + + if (Opcode == TargetOpcode::G_ANYEXT) { + // G_ANYEXT (G_IMPLICIT_DEF) -> G_IMPLICIT_DEF + if (isInstUnsupported({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) + return false; + LLVM_DEBUG(dbgs() << ".. Combine G_ANYEXT(G_IMPLICIT_DEF): " << MI;); + Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, {DstReg}, {}); + } else { + // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top + // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT. + if (isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + return false; + LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;); + Builder.buildConstant(DstReg, 0); + } + markInstAndDefDead(MI, *DefMI, DeadInsts); return true; } @@ -139,8 +169,20 @@ public: return false; unsigned NumDefs = MI.getNumOperands() - 1; - MachineInstr *MergeI = getOpcodeDef(TargetOpcode::G_MERGE_VALUES, - MI.getOperand(NumDefs).getReg(), MRI); + + unsigned MergingOpcode; + LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg()); + LLT DestTy = MRI.getType(MI.getOperand(0).getReg()); + if (OpTy.isVector() && DestTy.isVector()) + MergingOpcode = TargetOpcode::G_CONCAT_VECTORS; + else if (OpTy.isVector() && !DestTy.isVector()) + MergingOpcode = TargetOpcode::G_BUILD_VECTOR; + else + MergingOpcode = TargetOpcode::G_MERGE_VALUES; + + MachineInstr *MergeI = + getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI); + if (!MergeI) return false; @@ -277,6 +319,19 @@ private: auto Step = LI.getAction(Query); return Step.Action == Unsupported || Step.Action == NotFound; } + + /// Looks through copy instructions and returns the actual + /// source register. + unsigned lookThroughCopyInstrs(unsigned Reg) { + unsigned TmpReg; + while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { + if (MRI.getType(TmpReg).isValid()) + Reg = TmpReg; + else + break; + } + return Reg; + } }; } // namespace llvm diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index d122e67b87b8..9b4ecf9284e3 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -32,6 +32,7 @@ namespace llvm { class LegalizerInfo; class Legalizer; class MachineRegisterInfo; +class GISelChangeObserver; class LegalizerHelper { public: @@ -48,7 +49,10 @@ public: UnableToLegalize, }; - LegalizerHelper(MachineFunction &MF); + LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, + MachineIRBuilder &B); + LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, + GISelChangeObserver &Observer, MachineIRBuilder &B); /// Replace \p MI by a sequence of legal instructions that can implement the /// same operation. Note that this means \p MI may be deleted, so any iterator @@ -87,7 +91,7 @@ public: /// Expose MIRBuilder so clients can set their own RecordInsertInstruction /// functions - MachineIRBuilder MIRBuilder; + MachineIRBuilder &MIRBuilder; /// Expose LegalizerInfo so the clients can re-use. const LegalizerInfo &getLegalizerInfo() const { return LI; } @@ -112,8 +116,12 @@ private: void extractParts(unsigned Reg, LLT Ty, int NumParts, SmallVectorImpl &VRegs); + LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + MachineRegisterInfo &MRI; const LegalizerInfo &LI; + /// To keep track of changes made by the LegalizerHelper. + GISelChangeObserver &Observer; }; /// Helper function that creates the given libcall. diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index a8c26082f221..13776dd3e87d 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -39,6 +39,7 @@ class MachineInstr; class MachineIRBuilder; class MachineRegisterInfo; class MCInstrInfo; +class GISelChangeObserver; namespace LegalizeActions { enum LegalizeAction : std::uint8_t { @@ -121,7 +122,7 @@ struct LegalityQuery { ArrayRef Types; struct MemDesc { - uint64_t Size; + uint64_t SizeInBits; AtomicOrdering Ordering; }; @@ -651,6 +652,20 @@ public: return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); } + /// Widen the scalar to match the size of another. + LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { + typeIdx(TypeIdx); + return widenScalarIf( + [=](const LegalityQuery &Query) { + return Query.Types[LargeTypeIdx].getScalarSizeInBits() > + Query.Types[TypeIdx].getSizeInBits(); + }, + [=](const LegalityQuery &Query) { + return std::make_pair(TypeIdx, + Query.Types[LargeTypeIdx].getElementType()); + }); + } + /// Add more elements to the vector to reach the next power of two. /// No effect if the type is not a vector or the element count is a power of /// two. @@ -693,6 +708,8 @@ public: }, [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; + if (MaxElements == 1) + return std::make_pair(TypeIdx, VecTy.getElementType()); return std::make_pair( TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); }); @@ -947,9 +964,9 @@ public: bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; - virtual bool legalizeCustom(MachineInstr &MI, - MachineRegisterInfo &MRI, - MachineIRBuilder &MIRBuilder) const; + virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const; private: /// Determine what action should be taken to legalize the given generic diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index ac1673de5f3f..37de8f030410 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H #define LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/LowLevelType.h" @@ -30,6 +31,7 @@ namespace llvm { class MachineFunction; class MachineInstr; class TargetInstrInfo; +class GISelChangeObserver; /// Class which stores all the state required in a MachineIRBuilder. /// Since MachineIRBuilders will only store state in this object, it allows @@ -50,62 +52,177 @@ struct MachineIRBuilderState { MachineBasicBlock::iterator II; /// @} - std::function InsertedInstr; + GISelChangeObserver *Observer; + + GISelCSEInfo *CSEInfo; }; -/// Helper class to build MachineInstr. -/// It keeps internally the insertion point and debug location for all -/// the new instructions we want to create. -/// This information can be modify via the related setters. -class MachineIRBuilderBase { +class DstOp { + union { + LLT LLTTy; + unsigned Reg; + const TargetRegisterClass *RC; + }; - MachineIRBuilderState State; - const TargetInstrInfo &getTII() { - assert(State.TII && "TargetInstrInfo is not set"); - return *State.TII; +public: + enum class DstType { Ty_LLT, Ty_Reg, Ty_RC }; + DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {} + DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {} + + void addDefToMIB(MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) const { + switch (Ty) { + case DstType::Ty_Reg: + MIB.addDef(Reg); + break; + case DstType::Ty_LLT: + MIB.addDef(MRI.createGenericVirtualRegister(LLTTy)); + break; + case DstType::Ty_RC: + MIB.addDef(MRI.createVirtualRegister(RC)); + break; + } } - void validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend); + LLT getLLTTy(const MachineRegisterInfo &MRI) const { + switch (Ty) { + case DstType::Ty_RC: + return LLT{}; + case DstType::Ty_LLT: + return LLTTy; + case DstType::Ty_Reg: + return MRI.getType(Reg); + } + llvm_unreachable("Unrecognised DstOp::DstType enum"); + } -protected: - unsigned getDestFromArg(unsigned Reg) { return Reg; } - unsigned getDestFromArg(LLT Ty) { - return getMF().getRegInfo().createGenericVirtualRegister(Ty); + unsigned getReg() const { + assert(Ty == DstType::Ty_Reg && "Not a register"); + return Reg; } - unsigned getDestFromArg(const TargetRegisterClass *RC) { - return getMF().getRegInfo().createVirtualRegister(RC); + + const TargetRegisterClass *getRegClass() const { + switch (Ty) { + case DstType::Ty_RC: + return RC; + default: + llvm_unreachable("Not a RC Operand"); + } } - void addUseFromArg(MachineInstrBuilder &MIB, unsigned Reg) { - MIB.addUse(Reg); + DstType getDstOpKind() const { return Ty; } + +private: + DstType Ty; +}; + +class SrcOp { + union { + MachineInstrBuilder SrcMIB; + unsigned Reg; + CmpInst::Predicate Pred; + }; + +public: + enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate }; + SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {} + SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {} + + void addSrcToMIB(MachineInstrBuilder &MIB) const { + switch (Ty) { + case SrcType::Ty_Predicate: + MIB.addPredicate(Pred); + break; + case SrcType::Ty_Reg: + MIB.addUse(Reg); + break; + case SrcType::Ty_MIB: + MIB.addUse(SrcMIB->getOperand(0).getReg()); + break; + } } - void addUseFromArg(MachineInstrBuilder &MIB, const MachineInstrBuilder &UseMIB) { - MIB.addUse(UseMIB->getOperand(0).getReg()); + LLT getLLTTy(const MachineRegisterInfo &MRI) const { + switch (Ty) { + case SrcType::Ty_Predicate: + llvm_unreachable("Not a register operand"); + case SrcType::Ty_Reg: + return MRI.getType(Reg); + case SrcType::Ty_MIB: + return MRI.getType(SrcMIB->getOperand(0).getReg()); + } + llvm_unreachable("Unrecognised SrcOp::SrcType enum"); } - void addUsesFromArgs(MachineInstrBuilder &MIB) { } - template - void addUsesFromArgs(MachineInstrBuilder &MIB, UseArgTy &&Arg1, UseArgsTy &&... Args) { - addUseFromArg(MIB, Arg1); - addUsesFromArgs(MIB, std::forward(Args)...); + unsigned getReg() const { + switch (Ty) { + case SrcType::Ty_Predicate: + llvm_unreachable("Not a register operand"); + case SrcType::Ty_Reg: + return Reg; + case SrcType::Ty_MIB: + return SrcMIB->getOperand(0).getReg(); + } + llvm_unreachable("Unrecognised SrcOp::SrcType enum"); } - unsigned getRegFromArg(unsigned Reg) { return Reg; } - unsigned getRegFromArg(const MachineInstrBuilder &MIB) { - return MIB->getOperand(0).getReg(); + + CmpInst::Predicate getPredicate() const { + switch (Ty) { + case SrcType::Ty_Predicate: + return Pred; + default: + llvm_unreachable("Not a register operand"); + } } - void validateBinaryOp(unsigned Res, unsigned Op0, unsigned Op1); + SrcType getSrcOpKind() const { return Ty; } + +private: + SrcType Ty; +}; + +class FlagsOp { + Optional Flags; + +public: + explicit FlagsOp(unsigned F) : Flags(F) {} + FlagsOp() : Flags(None) {} + Optional getFlags() const { return Flags; } +}; +/// Helper class to build MachineInstr. +/// It keeps internally the insertion point and debug location for all +/// the new instructions we want to create. +/// This information can be modify via the related setters. +class MachineIRBuilder { + + MachineIRBuilderState State; + +protected: + void validateTruncExt(const LLT &Dst, const LLT &Src, bool IsExtend); + + void validateBinaryOp(const LLT &Res, const LLT &Op0, const LLT &Op1); + + void validateSelectOp(const LLT &ResTy, const LLT &TstTy, const LLT &Op0Ty, + const LLT &Op1Ty); + void recordInsertion(MachineInstr *MI) const; public: /// Some constructors for easy use. - MachineIRBuilderBase() = default; - MachineIRBuilderBase(MachineFunction &MF) { setMF(MF); } - MachineIRBuilderBase(MachineInstr &MI) : MachineIRBuilderBase(*MI.getMF()) { + MachineIRBuilder() = default; + MachineIRBuilder(MachineFunction &MF) { setMF(MF); } + MachineIRBuilder(MachineInstr &MI) : MachineIRBuilder(*MI.getMF()) { setInstr(MI); } - MachineIRBuilderBase(const MachineIRBuilderState &BState) : State(BState) {} + virtual ~MachineIRBuilder() = default; + + MachineIRBuilder(const MachineIRBuilderState &BState) : State(BState) {} + + const TargetInstrInfo &getTII() { + assert(State.TII && "TargetInstrInfo is not set"); + return *State.TII; + } /// Getter for the function we currently build. MachineFunction &getMF() { @@ -118,16 +235,25 @@ public: /// Getter for MRI MachineRegisterInfo *getMRI() { return State.MRI; } + const MachineRegisterInfo *getMRI() const { return State.MRI; } /// Getter for the State MachineIRBuilderState &getState() { return State; } /// Getter for the basic block we currently build. - MachineBasicBlock &getMBB() { + const MachineBasicBlock &getMBB() const { assert(State.MBB && "MachineBasicBlock is not set"); return *State.MBB; } + MachineBasicBlock &getMBB() { + return const_cast( + const_cast(this)->getMBB()); + } + + GISelCSEInfo *getCSEInfo() { return State.CSEInfo; } + const GISelCSEInfo *getCSEInfo() const { return State.CSEInfo; } + /// Current insertion point for new instructions. MachineBasicBlock::iterator getInsertPt() { return State.II; } @@ -137,10 +263,12 @@ public: void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II); /// @} + void setCSEInfo(GISelCSEInfo *Info); + /// \name Setters for the insertion point. /// @{ /// Set the MachineFunction where to build instructions. - void setMF(MachineFunction &); + void setMF(MachineFunction &MF); /// Set the insertion point to the end of \p MBB. /// \pre \p MBB must be contained by getMF(). @@ -151,12 +279,8 @@ public: void setInstr(MachineInstr &MI); /// @} - /// \name Control where instructions we create are recorded (typically for - /// visiting again later during legalization). - /// @{ - void recordInsertion(MachineInstr *InsertedInstr) const; - void recordInsertions(std::function InsertedInstr); - void stopRecordingInsertions(); + void setChangeObserver(GISelChangeObserver &Observer); + void stopObservingChanges(); /// @} /// Set the debug location to \p DL for all the next build instructions. @@ -208,6 +332,10 @@ public: const MDNode *Variable, const MDNode *Expr); + /// Build and insert a DBG_LABEL instructions specifying that \p Label is + /// given. Convert "llvm.dbg.label Label" to "DBG_LABEL Label". + MachineInstrBuilder buildDbgLabel(const MDNode *Label); + /// Build and insert \p Res = G_FRAME_INDEX \p Idx /// /// G_FRAME_INDEX materializes the address of an alloca value or other @@ -296,9 +424,9 @@ public: /// registers with the same scalar type (typically s1) /// /// \return The newly created instruction. - MachineInstrBuilder buildUAdde(unsigned Res, unsigned CarryOut, unsigned Op0, - unsigned Op1, unsigned CarryIn); - + MachineInstrBuilder buildUAdde(const DstOp &Res, const DstOp &CarryOut, + const SrcOp &Op0, const SrcOp &Op1, + const SrcOp &CarryIn); /// Build and insert \p Res = G_ANYEXT \p Op0 /// @@ -314,11 +442,7 @@ public: /// /// \return The newly created instruction. - MachineInstrBuilder buildAnyExt(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildAnyExt(DstType &&Res, ArgType &&Arg) { - return buildAnyExt(getDestFromArg(Res), getRegFromArg(Arg)); - } + MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_SEXT \p Op /// @@ -332,11 +456,7 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. - template - MachineInstrBuilder buildSExt(DstType &&Res, ArgType &&Arg) { - return buildSExt(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildSExt(unsigned Res, unsigned Op); + MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_ZEXT \p Op /// @@ -350,11 +470,7 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. - template - MachineInstrBuilder buildZExt(DstType &&Res, ArgType &&Arg) { - return buildZExt(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildZExt(unsigned Res, unsigned Op); + MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -364,11 +480,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildSExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildSExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildSExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_ZEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -378,11 +490,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildZExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildZExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildZExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op); // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -392,11 +500,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildAnyExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildAnyExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildAnyExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = \p ExtOpc, \p Res = G_TRUNC \p /// Op, or \p Res = COPY \p Op depending on the differing sizes of \p Res and @@ -407,15 +511,11 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, unsigned Res, - unsigned Op); + MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, + const SrcOp &Op); /// Build and insert an appropriate cast between two registers of equal size. - template - MachineInstrBuilder buildCast(DstType &&Res, ArgType &&Arg) { - return buildCast(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildCast(unsigned Dst, unsigned Src); + MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src); /// Build and insert G_BR \p Dest /// @@ -460,7 +560,8 @@ public: /// type. /// /// \return The newly created instruction. - MachineInstrBuilder buildConstant(unsigned Res, const ConstantInt &Val); + virtual MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val); /// Build and insert \p Res = G_CONSTANT \p Val /// @@ -470,12 +571,8 @@ public: /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildConstant(unsigned Res, int64_t Val); + MachineInstrBuilder buildConstant(const DstOp &Res, int64_t Val); - template - MachineInstrBuilder buildConstant(DstType &&Res, int64_t Val) { - return buildConstant(getDestFromArg(Res), Val); - } /// Build and insert \p Res = G_FCONSTANT \p Val /// /// G_FCONSTANT is a floating-point constant with the specified size and @@ -485,17 +582,10 @@ public: /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildFConstant(DstType &&Res, const ConstantFP &Val) { - return buildFConstant(getDestFromArg(Res), Val); - } - MachineInstrBuilder buildFConstant(unsigned Res, const ConstantFP &Val); + virtual MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val); - template - MachineInstrBuilder buildFConstant(DstType &&Res, double Val) { - return buildFConstant(getDestFromArg(Res), Val); - } - MachineInstrBuilder buildFConstant(unsigned Res, double Val); + MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); /// Build and insert \p Res = COPY Op /// @@ -504,11 +594,7 @@ public: /// \pre setBasicBlock or setMI must have been called. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildCopy(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildCopy(DstType &&Res, SrcType &&Src) { - return buildCopy(getDestFromArg(Res), getRegFromArg(Src)); - } + MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op); /// Build and insert `Res = G_LOAD Addr, MMO`. /// @@ -555,10 +641,7 @@ public: MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index); /// Build and insert \p Res = IMPLICIT_DEF. - template MachineInstrBuilder buildUndef(DstType &&Res) { - return buildUndef(getDestFromArg(Res)); - } - MachineInstrBuilder buildUndef(unsigned Res); + MachineInstrBuilder buildUndef(const DstOp &Res); /// Build and insert instructions to put \p Ops together at the specified p /// Indices to form a larger register. @@ -587,7 +670,7 @@ public: /// \pre The type of all \p Ops registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMerge(unsigned Res, ArrayRef Ops); + MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef Ops); /// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op /// @@ -599,7 +682,50 @@ public: /// \pre The type of all \p Res registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildUnmerge(ArrayRef Res, unsigned Op); + MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + + /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... + /// + /// G_BUILD_VECTOR creates a vector value from multiple scalar registers. + /// \pre setBasicBlock or setMI must have been called. + /// \pre The entire register \p Res (and no more) must be covered by the + /// input scalar registers. + /// \pre The type of all \p Ops registers must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBuildVector(const DstOp &Res, + ArrayRef Ops); + + /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... + /// + /// G_BUILD_VECTOR_TRUNC creates a vector value from multiple scalar registers + /// which have types larger than the destination vector element type, and + /// truncates the values to fit. + /// + /// If the operands given are already the same size as the vector elt type, + /// then this method will instead create a G_BUILD_VECTOR instruction. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre The type of all \p Ops registers must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res, + ArrayRef Ops); + + /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... + /// + /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more + /// vectors. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre The entire register \p Res (and no more) must be covered by the input + /// registers. + /// \pre The type of all source operands must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildConcatVectors(const DstOp &Res, + ArrayRef Ops); MachineInstrBuilder buildInsert(unsigned Res, unsigned Src, unsigned Op, unsigned Index); @@ -627,11 +753,7 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. - template - MachineInstrBuilder buildFPTrunc(DstType &&Res, SrcType &&Src) { - return buildFPTrunc(getDestFromArg(Res), getRegFromArg(Src)); - } - MachineInstrBuilder buildFPTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_TRUNC \p Op /// @@ -644,11 +766,7 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. - MachineInstrBuilder buildTrunc(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildTrunc(DstType &&Res, SrcType &&Src) { - return buildTrunc(getDestFromArg(Res), getRegFromArg(Src)); - } + MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert a \p Res = G_ICMP \p Pred, \p Op0, \p Op1 /// @@ -662,8 +780,8 @@ public: /// \pre \p Pred must be an integer predicate. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, - unsigned Res, unsigned Op0, unsigned Op1); + MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert a \p Res = G_FCMP \p Pred\p Op0, \p Op1 /// @@ -677,8 +795,8 @@ public: /// \pre \p Pred must be a floating-point predicate. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, - unsigned Res, unsigned Op0, unsigned Op1); + MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert a \p Res = G_SELECT \p Tst, \p Op0, \p Op1 /// @@ -690,8 +808,8 @@ public: /// elements as the other parameters. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSelect(unsigned Res, unsigned Tst, - unsigned Op0, unsigned Op1); + MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert \p Res = G_INSERT_VECTOR_ELT \p Val, /// \p Elt, \p Idx @@ -703,8 +821,10 @@ public: /// with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildInsertVectorElement(unsigned Res, unsigned Val, - unsigned Elt, unsigned Idx); + MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, + const SrcOp &Val, + const SrcOp &Elt, + const SrcOp &Idx); /// Build and insert \p Res = G_EXTRACT_VECTOR_ELT \p Val, \p Idx /// @@ -714,8 +834,9 @@ public: /// \pre \p Idx must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val, - unsigned Idx); + MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, + const SrcOp &Val, + const SrcOp &Idx); /// Build and insert `OldValRes, SuccessRes = /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`. @@ -952,19 +1073,7 @@ public: /// /// \return The newly created instruction. MachineInstrBuilder buildBlockAddress(unsigned Res, const BlockAddress *BA); -}; - -/// A CRTP class that contains methods for building instructions that can -/// be constant folded. MachineIRBuilders that want to inherit from this will -/// need to implement buildBinaryOp (for constant folding binary ops). -/// Alternatively, they can implement buildInstr(Opc, Dst, Uses...) to perform -/// additional folding for Opc. -template -class FoldableInstructionsBuilder : public MachineIRBuilderBase { - Base &base() { return static_cast(*this); } -public: - using MachineIRBuilderBase::MachineIRBuilderBase; /// Build and insert \p Res = G_ADD \p Op0, \p Op1 /// /// G_ADD sets \p Res to the sum of integer parameters \p Op0 and \p Op1, @@ -976,13 +1085,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAdd(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_ADD, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildAdd(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_ADD, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_SUB \p Op0, \p Op1 @@ -996,13 +1102,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSub(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_SUB, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildSub(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildSub(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_SUB, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_MUL \p Op0, \p Op1 @@ -1015,13 +1118,10 @@ public: /// with the same (scalar or vector) type). /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMul(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_MUL, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildMul(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildMul(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_AND \p Op0, \p Op1 @@ -1035,13 +1135,9 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAnd(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_AND, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildAnd(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildAnd(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_AND, {Dst}, {Src0, Src1}); } /// Build and insert \p Res = G_OR \p Op0, \p Op1 @@ -1054,39 +1150,14 @@ public: /// with the same (scalar or vector) type). /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildOr(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_OR, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildOr(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildOr(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}); } -}; -class MachineIRBuilder : public FoldableInstructionsBuilder { -public: - using FoldableInstructionsBuilder< - MachineIRBuilder>::FoldableInstructionsBuilder; - MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, - unsigned Src0, unsigned Src1) { - validateBinaryOp(Dst, Src0, Src1); - return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); - } - using FoldableInstructionsBuilder::buildInstr; - /// DAG like Generic method for building arbitrary instructions as above. - /// \Opc opcode for the instruction. - /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst - /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder) - /// Uses of type MachineInstrBuilder will perform - /// getOperand(0).getReg() to convert to register. - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, - UseArgsTy &&... Args) { - auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - addUsesFromArgs(MIB, std::forward(Args)...); - return MIB; - } + virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flags = None); }; } // End namespace llvm. diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index 82fd7eddb68a..c33b32b2db40 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -103,8 +103,8 @@ public: /// Currently the TableGen-like file would look like: /// \code /// PartialMapping[] = { - /// /*32-bit add*/ {0, 32, GPR}, - /// /*2x32-bit add*/ {0, 32, GPR}, {0, 32, GPR}, // <-- Same entry 3x + /// /*32-bit add*/ {0, 32, GPR}, // Scalar entry repeated for first vec elt. + /// /*2x32-bit add*/ {0, 32, GPR}, {32, 32, GPR}, /// /*<2x32-bit> vadd {0, 64, VPR} /// }; // PartialMapping duplicated. /// @@ -118,14 +118,15 @@ public: /// With the array of pointer, we would have: /// \code /// PartialMapping[] = { - /// /*32-bit add*/ {0, 32, GPR}, + /// /*32-bit add lower */ {0, 32, GPR}, + /// /*32-bit add upper */ {32, 32, GPR}, /// /*<2x32-bit> vadd {0, 64, VPR} /// }; // No more duplication. /// /// BreakDowns[] = { /// /*AddBreakDown*/ &PartialMapping[0], - /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[0], - /// /*VAddBreakDown*/ &PartialMapping[1] + /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[1], + /// /*VAddBreakDown*/ &PartialMapping[2] /// }; // Addresses of PartialMapping duplicated (smaller). /// /// ValueMapping[] { diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h index 51e3a2732972..82b791d35b2b 100644 --- a/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/include/llvm/CodeGen/GlobalISel/Utils.h @@ -108,5 +108,8 @@ APFloat getAPFloatFromSize(double Val, unsigned Size); /// fallback. void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); +Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, + const unsigned Op2, + const MachineRegisterInfo &MRI); } // End namespace llvm. #endif diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 80bd796d5374..9c918ae1104f 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -70,7 +70,7 @@ namespace ISD { /// of the frame or return address to return. An index of zero corresponds /// to the current function's frame or return address, an index of one to /// the parent's frame or return address, and so on. - FRAMEADDR, RETURNADDR, ADDROFRETURNADDR, + FRAMEADDR, RETURNADDR, ADDROFRETURNADDR, SPONENTRY, /// LOCAL_RECOVER - Represents the llvm.localrecover intrinsic. /// Materializes the offset from the local object pointer of another @@ -256,6 +256,29 @@ namespace ISD { /// Same for multiplication. SMULO, UMULO, + /// RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 + /// integers with the same bit width (W). If the true value of LHS + RHS + /// exceeds the largest value that can be represented by W bits, the + /// resulting value is this maximum value. Otherwise, if this value is less + /// than the smallest value that can be represented by W bits, the + /// resulting value is this minimum value. + SADDSAT, UADDSAT, + + /// RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 + /// integers with the same bit width (W). If the true value of LHS - RHS + /// exceeds the largest value that can be represented by W bits, the + /// resulting value is this maximum value. Otherwise, if this value is less + /// than the smallest value that can be represented by W bits, the + /// resulting value is this minimum value. + SSUBSAT, USUBSAT, + + /// RESULT = SMULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on + /// 2 integers with the same width and scale. SCALE represents the scale of + /// both operands as fixed point numbers. This SCALE parameter must be a + /// constant integer. A scale of zero is effectively performing + /// multiplication on 2 integers. + SMULFIX, + /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, @@ -272,7 +295,8 @@ namespace ISD { /// They are used to limit optimizations while the DAG is being optimized. STRICT_FSQRT, STRICT_FPOW, STRICT_FPOWI, STRICT_FSIN, STRICT_FCOS, STRICT_FEXP, STRICT_FEXP2, STRICT_FLOG, STRICT_FLOG10, STRICT_FLOG2, - STRICT_FRINT, STRICT_FNEARBYINT, + STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, + STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, /// FMA - Perform a * b + c with no intermediate rounding step. FMA, @@ -377,9 +401,13 @@ namespace ISD { /// When the 1st operand is a vector, the shift amount must be in the same /// type. (TLI.getShiftAmountTy() will return the same type when the input /// type is a vector.) - /// For rotates, the shift amount is treated as an unsigned amount modulo - /// the element size of the first operand. - SHL, SRA, SRL, ROTL, ROTR, + /// For rotates and funnel shifts, the shift amount is treated as an unsigned + /// amount modulo the element size of the first operand. + /// + /// Funnel 'double' shifts take 3 operands, 2 inputs and the shift amount. + /// fshl(X,Y,Z): (X << (Z % BW)) | (Y >> (BW - (Z % BW))) + /// fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW)) + SHL, SRA, SRL, ROTL, ROTR, FSHL, FSHR, /// Byte Swap and Counting operators. BSWAP, CTTZ, CTLZ, CTPOP, BITREVERSE, @@ -461,31 +489,33 @@ namespace ISD { /// in-register any-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is any-extended into the corresponding, wider result - /// elements with the high bits becoming undef. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is any-extended into the corresponding, + /// wider result elements with the high bits becoming undef. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. ANY_EXTEND_VECTOR_INREG, /// SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an /// in-register sign-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is sign-extended into the corresponding, wider result - /// elements. - // FIXME: The SIGN_EXTEND_INREG node isn't specifically limited to - // scalars, but it also doesn't handle vectors well. Either it should be - // restricted to scalars or this node (and its handling) should be merged - // into it. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is sign-extended into the corresponding, + /// wider result elements. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. SIGN_EXTEND_VECTOR_INREG, /// ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an /// in-register zero-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is zero-extended into the corresponding, wider result - /// elements. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is zero-extended into the corresponding, + /// wider result elements. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. ZERO_EXTEND_VECTOR_INREG, /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned @@ -550,22 +580,29 @@ namespace ISD { /// is often a storage-only type but has native conversions. FP16_TO_FP, FP_TO_FP16, - /// FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, - /// FLOG, FLOG2, FLOG10, FEXP, FEXP2, - /// FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary - /// floating point operations. These are inspired by libm. - FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, + /// Perform various unary floating-point operations inspired by libm. + FNEG, FABS, FSQRT, FCBRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR, /// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two /// values. - /// In the case where a single input is NaN, the non-NaN input is returned. + // + /// In the case where a single input is a NaN (either signaling or quiet), + /// the non-NaN input is returned. /// /// The return value of (FMINNUM 0.0, -0.0) could be either 0.0 or -0.0. FMINNUM, FMAXNUM, - /// FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that - /// when a single input is NaN, NaN is returned. - FMINNAN, FMAXNAN, + + /// FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on + /// two values, following the IEEE-754 2008 definition. This differs from + /// FMINNUM/FMAXNUM in the handling of signaling NaNs. If one input is a + /// signaling NaN, returns a quiet NaN. + FMINNUM_IEEE, FMAXNUM_IEEE, + + /// FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 + /// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008 + /// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2018 draft semantics. + FMINIMUM, FMAXIMUM, /// FSINCOS - Compute both fsin and fcos as a single operation. FSINCOS, @@ -786,11 +823,20 @@ namespace ISD { // Masked load and store - consecutive vector load and store operations // with additional mask operand that prevents memory accesses to the // masked-off lanes. + // + // Val, OutChain = MLOAD(BasePtr, Mask, PassThru) + // OutChain = MSTORE(Value, BasePtr, Mask) MLOAD, MSTORE, // Masked gather and scatter - load and store operations for a vector of // random addresses with additional mask operand that prevents memory // accesses to the masked-off lanes. + // + // Val, OutChain = GATHER(InChain, PassThru, Mask, BasePtr, Index, Scale) + // OutChain = SCATTER(InChain, Value, Mask, BasePtr, Index, Scale) + // + // The Index operand can have more vector elements than the other operands + // due to type legalization. The extra elements are ignored. MGATHER, MSCATTER, /// This corresponds to the llvm.lifetime.* intrinsics. The first operand diff --git a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h index c3046da90b8d..38fcb37b1e69 100644 --- a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h +++ b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h @@ -15,7 +15,7 @@ #ifndef LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H #define LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H -#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/BuiltinGCs.h" #include namespace { diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h index fee131e4a3c6..18c13ca8f598 100644 --- a/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -15,7 +15,7 @@ #ifndef LLVM_CODEGEN_LINKALLCODEGENCOMPONENTS_H #define LLVM_CODEGEN_LINKALLCODEGENCOMPONENTS_H -#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/BuiltinGCs.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/Target/TargetMachine.h" @@ -36,11 +36,7 @@ namespace { (void) llvm::createGreedyRegisterAllocator(); (void) llvm::createDefaultPBQPRegisterAllocator(); - llvm::linkCoreCLRGC(); - llvm::linkOcamlGC(); - llvm::linkErlangGC(); - llvm::linkShadowStackGC(); - llvm::linkStatepointExampleGC(); + llvm::linkAllBuiltinGCs(); (void) llvm::createBURRListDAGScheduler(nullptr, llvm::CodeGenOpt::Default); diff --git a/include/llvm/CodeGen/LiveIntervals.h b/include/llvm/CodeGen/LiveIntervals.h index 291a07a712cb..16ab1dc475c4 100644 --- a/include/llvm/CodeGen/LiveIntervals.h +++ b/include/llvm/CodeGen/LiveIntervals.h @@ -198,10 +198,10 @@ class VirtRegMap; void pruneValue(LiveRange &LR, SlotIndex Kill, SmallVectorImpl *EndPoints); - /// This function should not be used. Its intend is to tell you that - /// you are doing something wrong if you call pruveValue directly on a + /// This function should not be used. Its intent is to tell you that you are + /// doing something wrong if you call pruneValue directly on a /// LiveInterval. Indeed, you are supposed to call pruneValue on the main - /// LiveRange and all the LiveRange of the subranges if any. + /// LiveRange and all the LiveRanges of the subranges if any. LLVM_ATTRIBUTE_UNUSED void pruneValue(LiveInterval &, SlotIndex, SmallVectorImpl *) { llvm_unreachable( diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h index 301a45066b4c..7312902e21b7 100644 --- a/include/llvm/CodeGen/LivePhysRegs.h +++ b/include/llvm/CodeGen/LivePhysRegs.h @@ -48,7 +48,8 @@ class raw_ostream; /// when walking backward/forward through a basic block. class LivePhysRegs { const TargetRegisterInfo *TRI = nullptr; - SparseSet LiveRegs; + using RegisterSet = SparseSet>; + RegisterSet LiveRegs; public: /// Constructs an unitialized set. init() needs to be called to initialize it. @@ -76,7 +77,7 @@ public: bool empty() const { return LiveRegs.empty(); } /// Adds a physical register and all its sub-registers to the set. - void addReg(unsigned Reg) { + void addReg(MCPhysReg Reg) { assert(TRI && "LivePhysRegs is not initialized."); assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); @@ -86,7 +87,7 @@ public: /// Removes a physical register, all its sub-registers, and all its /// super-registers from the set. - void removeReg(unsigned Reg) { + void removeReg(MCPhysReg Reg) { assert(TRI && "LivePhysRegs is not initialized."); assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); for (MCRegAliasIterator R(Reg, TRI, true); R.isValid(); ++R) @@ -95,7 +96,7 @@ public: /// Removes physical registers clobbered by the regmask operand \p MO. void removeRegsInMask(const MachineOperand &MO, - SmallVectorImpl> *Clobbers = + SmallVectorImpl> *Clobbers = nullptr); /// Returns true if register \p Reg is contained in the set. This also @@ -103,10 +104,10 @@ public: /// addReg() always adds all sub-registers to the set as well. /// Note: Returns false if just some sub registers are live, use available() /// when searching a free register. - bool contains(unsigned Reg) const { return LiveRegs.count(Reg); } + bool contains(MCPhysReg Reg) const { return LiveRegs.count(Reg); } /// Returns true if register \p Reg and no aliasing register is in the set. - bool available(const MachineRegisterInfo &MRI, unsigned Reg) const; + bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const; /// Remove defined registers and regmask kills from the set. void removeDefs(const MachineInstr &MI); @@ -126,7 +127,7 @@ public: /// defined or clobbered by a regmask. The operand will identify whether this /// is a regmask or register operand. void stepForward(const MachineInstr &MI, - SmallVectorImpl> &Clobbers); + SmallVectorImpl> &Clobbers); /// Adds all live-in registers of basic block \p MBB. /// Live in registers are the registers in the blocks live-in list and the @@ -143,7 +144,7 @@ public: /// registers. void addLiveOutsNoPristines(const MachineBasicBlock &MBB); - using const_iterator = SparseSet::const_iterator; + using const_iterator = RegisterSet::const_iterator; const_iterator begin() const { return LiveRegs.begin(); } const_iterator end() const { return LiveRegs.end(); } diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h index 249545906e01..5e9dd8b3cdf6 100644 --- a/include/llvm/CodeGen/LiveRegUnits.h +++ b/include/llvm/CodeGen/LiveRegUnits.h @@ -85,14 +85,14 @@ public: bool empty() const { return Units.none(); } /// Adds register units covered by physical register \p Reg. - void addReg(unsigned Reg) { + void addReg(MCPhysReg Reg) { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) Units.set(*Unit); } /// Adds register units covered by physical register \p Reg that are /// part of the lanemask \p Mask. - void addRegMasked(unsigned Reg, LaneBitmask Mask) { + void addRegMasked(MCPhysReg Reg, LaneBitmask Mask) { for (MCRegUnitMaskIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) { LaneBitmask UnitMask = (*Unit).second; if (UnitMask.none() || (UnitMask & Mask).any()) @@ -101,7 +101,7 @@ public: } /// Removes all register units covered by physical register \p Reg. - void removeReg(unsigned Reg) { + void removeReg(MCPhysReg Reg) { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) Units.reset(*Unit); } @@ -115,7 +115,7 @@ public: void addRegsInMask(const uint32_t *RegMask); /// Returns true if no part of physical register \p Reg is live. - bool available(unsigned Reg) const { + bool available(MCPhysReg Reg) const { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) { if (Units.test(*Unit)) return false; diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h index 7f46406c4789..98ac81915dc0 100644 --- a/include/llvm/CodeGen/MIRYamlMapping.h +++ b/include/llvm/CodeGen/MIRYamlMapping.h @@ -425,6 +425,7 @@ struct MachineFrameInfo { StringValue StackProtector; // TODO: Serialize FunctionContextIdx unsigned MaxCallFrameSize = ~0u; ///< ~0u means: not computed yet. + unsigned CVBytesOfCalleeSavedRegisters = 0; bool HasOpaqueSPAdjustment = false; bool HasVAStart = false; bool HasMustTailInVarArgFunc = false; @@ -443,6 +444,8 @@ struct MachineFrameInfo { AdjustsStack == Other.AdjustsStack && HasCalls == Other.HasCalls && StackProtector == Other.StackProtector && MaxCallFrameSize == Other.MaxCallFrameSize && + CVBytesOfCalleeSavedRegisters == + Other.CVBytesOfCalleeSavedRegisters && HasOpaqueSPAdjustment == Other.HasOpaqueSPAdjustment && HasVAStart == Other.HasVAStart && HasMustTailInVarArgFunc == Other.HasMustTailInVarArgFunc && @@ -465,6 +468,8 @@ template <> struct MappingTraits { YamlIO.mapOptional("stackProtector", MFI.StackProtector, StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize, (unsigned)~0); + YamlIO.mapOptional("cvBytesOfCalleeSavedRegisters", + MFI.CVBytesOfCalleeSavedRegisters, 0U); YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment, false); YamlIO.mapOptional("hasVAStart", MFI.HasVAStart, false); @@ -489,6 +494,7 @@ struct MachineFunction { bool FailedISel = false; // Register information bool TracksRegLiveness = false; + bool HasWinCFI = false; std::vector VirtualRegisters; std::vector LiveIns; Optional> CalleeSavedRegisters; @@ -512,6 +518,7 @@ template <> struct MappingTraits { YamlIO.mapOptional("selected", MF.Selected, false); YamlIO.mapOptional("failedISel", MF.FailedISel, false); YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness, false); + YamlIO.mapOptional("hasWinCFI", MF.HasWinCFI, false); YamlIO.mapOptional("registers", MF.VirtualRegisters, std::vector()); YamlIO.mapOptional("liveins", MF.LiveIns, diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index ace33efd8713..ec2f270fcb3f 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -569,6 +569,12 @@ public: return !empty() && back().isReturn(); } + /// Convenience function that returns true if the bock ends in a EH scope + /// return instruction. + bool isEHScopeReturnBlock() const { + return !empty() && back().isEHScopeReturn(); + } + /// Split the critical edge from this block to the given successor block, and /// return the newly created block, or null if splitting is not possible. /// diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index 2d6081f3577d..c2706a21a177 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -28,9 +28,14 @@ class AllocaInst; /// The CalleeSavedInfo class tracks the information need to locate where a /// callee saved register is in the current frame. +/// Callee saved reg can also be saved to a different register rather than +/// on the stack by setting DstReg instead of FrameIdx. class CalleeSavedInfo { unsigned Reg; - int FrameIdx; + union { + int FrameIdx; + unsigned DstReg; + }; /// Flag indicating whether the register is actually restored in the epilog. /// In most cases, if a register is saved, it is also restored. There are /// some situations, though, when this is not the case. For example, the @@ -44,17 +49,29 @@ class CalleeSavedInfo { /// by implicit uses on the return instructions, however, the required /// changes in the ARM backend would be quite extensive. bool Restored; + /// Flag indicating whether the register is spilled to stack or another + /// register. + bool SpilledToReg; public: explicit CalleeSavedInfo(unsigned R, int FI = 0) - : Reg(R), FrameIdx(FI), Restored(true) {} + : Reg(R), FrameIdx(FI), Restored(true), SpilledToReg(false) {} // Accessors. unsigned getReg() const { return Reg; } int getFrameIdx() const { return FrameIdx; } - void setFrameIdx(int FI) { FrameIdx = FI; } + unsigned getDstReg() const { return DstReg; } + void setFrameIdx(int FI) { + FrameIdx = FI; + SpilledToReg = false; + } + void setDstReg(unsigned SpillReg) { + DstReg = SpillReg; + SpilledToReg = true; + } bool isRestored() const { return Restored; } void setRestored(bool R) { Restored = R; } + bool isSpilledToReg() const { return SpilledToReg; } }; /// The MachineFrameInfo class represents an abstract stack frame until @@ -266,10 +283,14 @@ private: /// It is only valid during and after prolog/epilog code insertion. unsigned MaxCallFrameSize = ~0u; + /// The number of bytes of callee saved registers that the target wants to + /// report for the current function in the CodeView S_FRAMEPROC record. + unsigned CVBytesOfCalleeSavedRegisters = 0; + /// The prolog/epilog code inserter fills in this vector with each - /// callee saved register saved in the frame. Beyond its use by the prolog/ - /// epilog code inserter, this data used for debug info and exception - /// handling. + /// callee saved register saved in either the frame or a different + /// register. Beyond its use by the prolog/ epilog code inserter, + /// this data is used for debug info and exception handling. std::vector CSInfo; /// Has CSInfo been set yet? @@ -603,6 +624,15 @@ public: } void setMaxCallFrameSize(unsigned S) { MaxCallFrameSize = S; } + /// Returns how many bytes of callee-saved registers the target pushed in the + /// prologue. Only used for debug info. + unsigned getCVBytesOfCalleeSavedRegisters() const { + return CVBytesOfCalleeSavedRegisters; + } + void setCVBytesOfCalleeSavedRegisters(unsigned S) { + CVBytesOfCalleeSavedRegisters = S; + } + /// Create a new object at a fixed location on the stack. /// All fixed objects should be created before other objects are created for /// efficiency. By default, fixed objects are not pointed to by LLVM IR diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index e8a4d529faac..25edf5bcce51 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -58,6 +58,7 @@ class DILocalVariable; class DILocation; class Function; class GlobalValue; +class LLVMTargetMachine; class MachineConstantPool; class MachineFrameInfo; class MachineFunction; @@ -70,7 +71,6 @@ class Pass; class PseudoSourceValueManager; class raw_ostream; class SlotIndexes; -class TargetMachine; class TargetRegisterClass; class TargetSubtargetInfo; struct WasmEHFuncInfo; @@ -225,7 +225,7 @@ struct LandingPadInfo { class MachineFunction { const Function &F; - const TargetMachine &Target; + const LLVMTargetMachine &Target; const TargetSubtargetInfo *STI; MCContext &Ctx; MachineModuleInfo &MMI; @@ -294,7 +294,7 @@ class MachineFunction { bool HasInlineAsm = false; /// True if any WinCFI instruction have been emitted in this function. - Optional HasWinCFI; + bool HasWinCFI = false; /// Current high-level properties of the IR of the function (e.g. is in SSA /// form or whether registers have been allocated) @@ -316,6 +316,9 @@ class MachineFunction { /// Map a landing pad's EH symbol to the call site indexes. DenseMap> LPadToCallSiteMap; + /// Map a landing pad to its index. + DenseMap WasmLPadToIndexMap; + /// Map of invoke call site index values to associated begin EH_LABEL. DenseMap CallSiteMap; @@ -363,10 +366,31 @@ public: int Slot, const DILocation *Loc) : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {} }; + + class Delegate { + virtual void anchor(); + + public: + virtual ~Delegate() = default; + /// Callback after an insertion. This should not modify the MI directly. + virtual void MF_HandleInsertion(MachineInstr &MI) = 0; + /// Callback before a removal. This should not modify the MI directly. + virtual void MF_HandleRemoval(MachineInstr &MI) = 0; + }; + +private: + Delegate *TheDelegate = nullptr; + + // Callbacks for insertion and removal. + void handleInsertion(MachineInstr &MI); + void handleRemoval(MachineInstr &MI); + friend struct ilist_traits; + +public: using VariableDbgInfoMapTy = SmallVector; VariableDbgInfoMapTy VariableDbgInfos; - MachineFunction(const Function &F, const TargetMachine &Target, + MachineFunction(const Function &F, const LLVMTargetMachine &Target, const TargetSubtargetInfo &STI, unsigned FunctionNum, MachineModuleInfo &MMI); MachineFunction(const MachineFunction &) = delete; @@ -379,6 +403,23 @@ public: init(); } + /// Reset the currently registered delegate - otherwise assert. + void resetDelegate(Delegate *delegate) { + assert(TheDelegate == delegate && + "Only the current delegate can perform reset!"); + TheDelegate = nullptr; + } + + /// Set the delegate. resetDelegate must be called before attempting + /// to set. + void setDelegate(Delegate *delegate) { + assert(delegate && !TheDelegate && + "Attempted to set delegate to null, or to change it without " + "first resetting it!"); + + TheDelegate = delegate; + } + MachineModuleInfo &getMMI() const { return MMI; } MCContext &getContext() const { return Ctx; } @@ -397,7 +438,7 @@ public: unsigned getFunctionNumber() const { return FunctionNumber; } /// getTarget - Return the target machine this machine code is compiled with - const TargetMachine &getTarget() const { return Target; } + const LLVMTargetMachine &getTarget() const { return Target; } /// getSubtarget - Return the subtarget for which this machine code is being /// compiled. @@ -484,8 +525,7 @@ public: } bool hasWinCFI() const { - assert(HasWinCFI.hasValue() && "HasWinCFI not set yet!"); - return *HasWinCFI; + return HasWinCFI; } void setHasWinCFI(bool v) { HasWinCFI = v; } @@ -619,6 +659,14 @@ public: BasicBlocks.sort(comp); } + /// Return the number of \p MachineInstrs in this \p MachineFunction. + unsigned getInstructionCount() const { + unsigned InstrCount = 0; + for (const MachineBasicBlock &MBB : BasicBlocks) + InstrCount += MBB.size(); + return InstrCount; + } + //===--------------------------------------------------------------------===// // Internal functions used to automatically number MachineBasicBlocks @@ -711,23 +759,14 @@ public: /// Allocate and initialize a register mask with @p NumRegister bits. uint32_t *allocateRegMask(); - /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand - /// pointers. This array is owned by the MachineFunction. - MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num); - - /// extractLoadMemRefs - Allocate an array and populate it with just the - /// load information from the given MachineMemOperand sequence. - std::pair - extractLoadMemRefs(MachineInstr::mmo_iterator Begin, - MachineInstr::mmo_iterator End); - - /// extractStoreMemRefs - Allocate an array and populate it with just the - /// store information from the given MachineMemOperand sequence. - std::pair - extractStoreMemRefs(MachineInstr::mmo_iterator Begin, - MachineInstr::mmo_iterator End); + /// Allocate and construct an extra info structure for a `MachineInstr`. + /// + /// This is allocated on the function's allocator and so lives the life of + /// the function. + MachineInstr::ExtraInfo * + createMIExtraInfo(ArrayRef MMOs, + MCSymbol *PreInstrSymbol = nullptr, + MCSymbol *PostInstrSymbol = nullptr); /// Allocate a string and populate it with the given external symbol name. const char *createExternalSymbolName(StringRef Name); @@ -776,7 +815,8 @@ public: LandingPadInfo &getOrCreateLandingPadInfo(MachineBasicBlock *LandingPad); /// Remap landing pad labels and remove any deleted landing pads. - void tidyLandingPads(DenseMap *LPMap = nullptr); + void tidyLandingPads(DenseMap *LPMap = nullptr, + bool TidyIfNoBeginLabels = true); /// Return a reference to the landing pad info for the current function. const std::vector &getLandingPads() const { @@ -788,7 +828,9 @@ public: void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel); - /// Add a new panding pad. Returns the label ID for the landing pad entry. + /// Add a new panding pad, and extract the exception handling information from + /// the landingpad instruction. Returns the label ID for the landing pad + /// entry. MCSymbol *addLandingPad(MachineBasicBlock *LandingPad); /// Provide the catch typeinfo for a landing pad. @@ -817,6 +859,22 @@ public: /// Map the landing pad's EH symbol to the call site indexes. void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef Sites); + /// Map the landing pad to its index. Used for Wasm exception handling. + void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index) { + WasmLPadToIndexMap[LPad] = Index; + } + + /// Returns true if the landing pad has an associate index in wasm EH. + bool hasWasmLandingPadIndex(const MachineBasicBlock *LPad) const { + return WasmLPadToIndexMap.count(LPad); + } + + /// Get the index in wasm EH for a given landing pad. + unsigned getWasmLandingPadIndex(const MachineBasicBlock *LPad) const { + assert(hasWasmLandingPadIndex(LPad)); + return WasmLPadToIndexMap.lookup(LPad); + } + /// Get the call site indexes for a landing pad EH symbol. SmallVectorImpl &getCallSiteLandingPad(MCSymbol *Sym) { assert(hasCallSiteLandingPad(Sym) && @@ -880,15 +938,6 @@ public: } }; -/// \name Exception Handling -/// \{ - -/// Extract the exception handling information from the landingpad instruction -/// and add them to the specified machine module info. -void addLandingPadInfo(const LandingPadInst &I, MachineBasicBlock &MBB); - -/// \} - //===--------------------------------------------------------------------===// // GraphTraits specializations for function basic block graphs (CFGs) //===--------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index 88e13cdf4138..ea1a2a536fc7 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -17,16 +17,20 @@ #define LLVM_CODEGEN_MACHINEINSTR_H #include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/PointerSumType.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/ArrayRecycler.h" +#include "llvm/Support/TrailingObjects.h" #include #include #include @@ -61,7 +65,7 @@ class MachineInstr : public ilist_node_with_parent> { public: - using mmo_iterator = MachineMemOperand **; + using mmo_iterator = ArrayRef::iterator; /// Flags to specify different kinds of comments to output in /// assembly code. These flags carry semantic information not @@ -93,8 +97,14 @@ public: // contraction operations like fma. FmAfn = 1 << 9, // Instruction may map to Fast math // instrinsic approximation. - FmReassoc = 1 << 10 // Instruction supports Fast math + FmReassoc = 1 << 10, // Instruction supports Fast math // reassociation of operand order. + NoUWrap = 1 << 11, // Instruction supports binary operator + // no unsigned wrap. + NoSWrap = 1 << 12, // Instruction supports binary operator + // no signed wrap. + IsExact = 1 << 13 // Instruction supports division is + // known to be exact. }; private: @@ -118,14 +128,102 @@ private: // anything other than to convey comment // information to AsmPrinter. - uint8_t NumMemRefs = 0; // Information on memory references. - // Note that MemRefs == nullptr, means 'don't know', not 'no memory access'. - // Calling code must treat missing information conservatively. If the number - // of memory operands required to be precise exceeds the maximum value of - // NumMemRefs - currently 256 - we remove the operands entirely. Note also - // that this is a non-owning reference to a shared copy on write buffer owned - // by the MachineFunction and created via MF.allocateMemRefsArray. - mmo_iterator MemRefs = nullptr; + /// Internal implementation detail class that provides out-of-line storage for + /// extra info used by the machine instruction when this info cannot be stored + /// in-line within the instruction itself. + /// + /// This has to be defined eagerly due to the implementation constraints of + /// `PointerSumType` where it is used. + class ExtraInfo final + : TrailingObjects { + public: + static ExtraInfo *create(BumpPtrAllocator &Allocator, + ArrayRef MMOs, + MCSymbol *PreInstrSymbol = nullptr, + MCSymbol *PostInstrSymbol = nullptr) { + bool HasPreInstrSymbol = PreInstrSymbol != nullptr; + bool HasPostInstrSymbol = PostInstrSymbol != nullptr; + auto *Result = new (Allocator.Allocate( + totalSizeToAlloc( + MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol), + alignof(ExtraInfo))) + ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol); + + // Copy the actual data into the trailing objects. + std::copy(MMOs.begin(), MMOs.end(), + Result->getTrailingObjects()); + + if (HasPreInstrSymbol) + Result->getTrailingObjects()[0] = PreInstrSymbol; + if (HasPostInstrSymbol) + Result->getTrailingObjects()[HasPreInstrSymbol] = + PostInstrSymbol; + + return Result; + } + + ArrayRef getMMOs() const { + return makeArrayRef(getTrailingObjects(), NumMMOs); + } + + MCSymbol *getPreInstrSymbol() const { + return HasPreInstrSymbol ? getTrailingObjects()[0] : nullptr; + } + + MCSymbol *getPostInstrSymbol() const { + return HasPostInstrSymbol + ? getTrailingObjects()[HasPreInstrSymbol] + : nullptr; + } + + private: + friend TrailingObjects; + + // Description of the extra info, used to interpret the actual optional + // data appended. + // + // Note that this is not terribly space optimized. This leaves a great deal + // of flexibility to fit more in here later. + const int NumMMOs; + const bool HasPreInstrSymbol; + const bool HasPostInstrSymbol; + + // Implement the `TrailingObjects` internal API. + size_t numTrailingObjects(OverloadToken) const { + return NumMMOs; + } + size_t numTrailingObjects(OverloadToken) const { + return HasPreInstrSymbol + HasPostInstrSymbol; + } + + // Just a boring constructor to allow us to initialize the sizes. Always use + // the `create` routine above. + ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol) + : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), + HasPostInstrSymbol(HasPostInstrSymbol) {} + }; + + /// Enumeration of the kinds of inline extra info available. It is important + /// that the `MachineMemOperand` inline kind has a tag value of zero to make + /// it accessible as an `ArrayRef`. + enum ExtraInfoInlineKinds { + EIIK_MMO = 0, + EIIK_PreInstrSymbol, + EIIK_PostInstrSymbol, + EIIK_OutOfLine + }; + + // We store extra information about the instruction here. The common case is + // expected to be nothing or a single pointer (typically a MMO or a symbol). + // We work to optimize this common case by storing it inline here rather than + // requiring a separate allocation, but we fall back to an allocation when + // multiple pointers are needed. + PointerSumType, + PointerSumTypeMember, + PointerSumTypeMember, + PointerSumTypeMember> + Info; DebugLoc debugLoc; // Source line information. @@ -310,7 +408,7 @@ public: /// Returns the opcode of this MachineInstr. unsigned getOpcode() const { return MCID->Opcode; } - /// Access to explicit operands of the instruction. + /// Retuns the total number of operands. unsigned getNumOperands() const { return NumOperands; } const MachineOperand& getOperand(unsigned i) const { @@ -412,28 +510,70 @@ public: return I - operands_begin(); } - /// Access to memory operands of the instruction - mmo_iterator memoperands_begin() const { return MemRefs; } - mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; } + /// Access to memory operands of the instruction. If there are none, that does + /// not imply anything about whether the function accesses memory. Instead, + /// the caller must behave conservatively. + ArrayRef memoperands() const { + if (!Info) + return {}; + + if (Info.is()) + return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); + + if (ExtraInfo *EI = Info.get()) + return EI->getMMOs(); + + return {}; + } + + /// Access to memory operands of the instruction. + /// + /// If `memoperands_begin() == memoperands_end()`, that does not imply + /// anything about whether the function accesses memory. Instead, the caller + /// must behave conservatively. + mmo_iterator memoperands_begin() const { return memoperands().begin(); } + + /// Access to memory operands of the instruction. + /// + /// If `memoperands_begin() == memoperands_end()`, that does not imply + /// anything about whether the function accesses memory. Instead, the caller + /// must behave conservatively. + mmo_iterator memoperands_end() const { return memoperands().end(); } + /// Return true if we don't have any memory operands which described the /// memory access done by this instruction. If this is true, calling code /// must be conservative. - bool memoperands_empty() const { return NumMemRefs == 0; } - - iterator_range memoperands() { - return make_range(memoperands_begin(), memoperands_end()); - } - iterator_range memoperands() const { - return make_range(memoperands_begin(), memoperands_end()); - } + bool memoperands_empty() const { return memoperands().empty(); } /// Return true if this instruction has exactly one MachineMemOperand. - bool hasOneMemOperand() const { - return NumMemRefs == 1; - } + bool hasOneMemOperand() const { return memoperands().size() == 1; } /// Return the number of memory operands. - unsigned getNumMemOperands() const { return NumMemRefs; } + unsigned getNumMemOperands() const { return memoperands().size(); } + + /// Helper to extract a pre-instruction symbol if one has been added. + MCSymbol *getPreInstrSymbol() const { + if (!Info) + return nullptr; + if (MCSymbol *S = Info.get()) + return S; + if (ExtraInfo *EI = Info.get()) + return EI->getPreInstrSymbol(); + + return nullptr; + } + + /// Helper to extract a post-instruction symbol if one has been added. + MCSymbol *getPostInstrSymbol() const { + if (!Info) + return nullptr; + if (MCSymbol *S = Info.get()) + return S; + if (ExtraInfo *EI = Info.get()) + return EI->getPostInstrSymbol(); + + return nullptr; + } /// API for querying MachineInstr properties. They are the same as MCInstrDesc /// queries but they are bundle aware. @@ -450,6 +590,8 @@ public: /// The second argument indicates whether the query should look inside /// instruction bundles. bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { + assert(MCFlag < 64 && + "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle."); // Inline the fast path for unbundled or bundle-internal instructions. if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) return getDesc().getFlags() & (1ULL << MCFlag); @@ -482,6 +624,12 @@ public: return hasProperty(MCID::Return, Type); } + /// Return true if this is an instruction that marks the end of an EH scope, + /// i.e., a catchpad or a cleanuppad instruction. + bool isEHScopeReturn(QueryType Type = AnyInBundle) const { + return hasProperty(MCID::EHScopeReturn, Type); + } + bool isCall(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Call, Type); } @@ -1323,47 +1471,63 @@ public: /// fewer operand than it started with. void RemoveOperand(unsigned OpNo); + /// Clear this MachineInstr's memory reference descriptor list. This resets + /// the memrefs to their most conservative state. This should be used only + /// as a last resort since it greatly pessimizes our knowledge of the memory + /// access performed by the instruction. + void dropMemRefs(MachineFunction &MF); + + /// Assign this MachineInstr's memory reference descriptor list. + /// + /// Unlike other methods, this *will* allocate them into a new array + /// associated with the provided `MachineFunction`. + void setMemRefs(MachineFunction &MF, ArrayRef MemRefs); + /// Add a MachineMemOperand to the machine instruction. /// This function should be used only occasionally. The setMemRefs function /// is the primary method for setting up a MachineInstr's MemRefs list. void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); - /// Assign this MachineInstr's memory reference descriptor list. - /// This does not transfer ownership. - void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { - setMemRefs(std::make_pair(NewMemRefs, NewMemRefsEnd-NewMemRefs)); - } + /// Clone another MachineInstr's memory reference descriptor list and replace + /// ours with it. + /// + /// Note that `*this` may be the incoming MI! + /// + /// Prefer this API whenever possible as it can avoid allocations in common + /// cases. + void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); - /// Assign this MachineInstr's memory reference descriptor list. First - /// element in the pair is the begin iterator/pointer to the array; the - /// second is the number of MemoryOperands. This does not transfer ownership - /// of the underlying memory. - void setMemRefs(std::pair NewMemRefs) { - MemRefs = NewMemRefs.first; - NumMemRefs = uint8_t(NewMemRefs.second); - assert(NumMemRefs == NewMemRefs.second && - "Too many memrefs - must drop memory operands"); - } + /// Clone the merge of multiple MachineInstrs' memory reference descriptors + /// list and replace ours with it. + /// + /// Note that `*this` may be one of the incoming MIs! + /// + /// Prefer this API whenever possible as it can avoid allocations in common + /// cases. + void cloneMergedMemRefs(MachineFunction &MF, + ArrayRef MIs); - /// Return a set of memrefs (begin iterator, size) which conservatively - /// describe the memory behavior of both MachineInstrs. This is appropriate - /// for use when merging two MachineInstrs into one. This routine does not - /// modify the memrefs of the this MachineInstr. - std::pair mergeMemRefsWith(const MachineInstr& Other); + /// Set a symbol that will be emitted just prior to the instruction itself. + /// + /// Setting this to a null pointer will remove any such symbol. + /// + /// FIXME: This is not fully implemented yet. + void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); + + /// Set a symbol that will be emitted just after the instruction itself. + /// + /// Setting this to a null pointer will remove any such symbol. + /// + /// FIXME: This is not fully implemented yet. + void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. uint16_t mergeFlagsWith(const MachineInstr& Other) const; - /// Clear this MachineInstr's memory reference descriptor list. This resets - /// the memrefs to their most conservative state. This should be used only - /// as a last resort since it greatly pessimizes our knowledge of the memory - /// access performed by the instruction. - void dropMemRefs() { - MemRefs = nullptr; - NumMemRefs = 0; - } + /// Copy all flags to MachineInst MIFlags + void copyIRFlags(const Instruction &I); /// Break any tie involving OpIdx. void untieRegOperand(unsigned OpIdx) { @@ -1377,6 +1541,13 @@ public: /// Add all implicit def and use operands to this instruction. void addImplicitDefUseOperands(MachineFunction &MF); + /// Scan instructions following MI and collect any matching DBG_VALUEs. + void collectDebugValues(SmallVectorImpl &DbgValues); + + /// Find all DBG_VALUEs immediately following this instruction that point + /// to a register def in this instruction and point them to \p Reg instead. + void changeDebugValuesDefReg(unsigned Reg); + private: /// If this instruction is embedded into a MachineFunction, return the /// MachineRegisterInfo object for the current function, otherwise @@ -1394,7 +1565,7 @@ private: void AddRegOperandsToUseLists(MachineRegisterInfo&); /// Slow path for hasProperty when we're dealing with a bundle. - bool hasPropertyInBundle(unsigned Mask, QueryType Type) const; + bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; /// Implements the logic of getRegClassConstraintEffectForVReg for the /// this MI and the given operand index \p OpIdx. diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index 665608755741..b5e523f655e7 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -191,15 +191,20 @@ public: return *this; } - const MachineInstrBuilder &setMemRefs(MachineInstr::mmo_iterator b, - MachineInstr::mmo_iterator e) const { - MI->setMemRefs(b, e); + const MachineInstrBuilder & + setMemRefs(ArrayRef MMOs) const { + MI->setMemRefs(*MF, MMOs); return *this; } - const MachineInstrBuilder &setMemRefs(std::pair MemOperandsRef) const { - MI->setMemRefs(MemOperandsRef); + const MachineInstrBuilder &cloneMemRefs(const MachineInstr &OtherMI) const { + MI->cloneMemRefs(*MF, OtherMI); + return *this; + } + + const MachineInstrBuilder & + cloneMergedMemRefs(ArrayRef OtherMIs) const { + MI->cloneMergedMemRefs(*MF, OtherMIs); return *this; } diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 554e89019b76..4371420bc7a2 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -46,10 +46,10 @@ namespace llvm { class BasicBlock; class CallInst; class Function; -class MachineFunction; +class LLVMTargetMachine; class MMIAddrLabelMap; +class MachineFunction; class Module; -class TargetMachine; //===----------------------------------------------------------------------===// /// This class can be derived from and used by targets to hold private @@ -76,7 +76,7 @@ protected: /// for specific use. /// class MachineModuleInfo : public ImmutablePass { - const TargetMachine &TM; + const LLVMTargetMachine &TM; /// This is the MCContext used for the entire code generator. MCContext Context; @@ -145,7 +145,7 @@ class MachineModuleInfo : public ImmutablePass { public: static char ID; // Pass identification, replacement for typeid - explicit MachineModuleInfo(const TargetMachine *TM = nullptr); + explicit MachineModuleInfo(const LLVMTargetMachine *TM = nullptr); ~MachineModuleInfo() override; // Initialization and Finalization diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h index 6a87fa2fbf00..17df1fa792b7 100644 --- a/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -80,6 +80,28 @@ public: SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } }; +/// MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation +/// for COFF targets. +class MachineModuleInfoCOFF : public MachineModuleInfoImpl { + /// GVStubs - These stubs are used to materialize global addresses in PIC + /// mode. + DenseMap GVStubs; + + virtual void anchor(); // Out of line virtual method. + +public: + MachineModuleInfoCOFF(const MachineModuleInfo &) {} + + StubValueTy &getGVStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return GVStubs[Sym]; + } + + /// Accessor methods to return the set of stubs in sorted order. + + SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } +}; + } // end namespace llvm #endif // LLVM_CODEGEN_MACHINEMODULEINFOIMPLS_H diff --git a/include/llvm/CodeGen/MachineOutliner.h b/include/llvm/CodeGen/MachineOutliner.h index 95bfc24b57ff..bfd1e994053a 100644 --- a/include/llvm/CodeGen/MachineOutliner.h +++ b/include/llvm/CodeGen/MachineOutliner.h @@ -61,9 +61,6 @@ public: /// \p OutlinedFunctions. unsigned FunctionIdx; - /// Set to false if the candidate overlapped with another candidate. - bool InCandidateList = true; - /// Identifier denoting the instructions to emit to call an outlined function /// from this point. Defined by the target. unsigned CallConstructionID; @@ -82,6 +79,12 @@ public: /// been used across the sequence. LiveRegUnits UsedInSequence; + /// Target-specific flags for this Candidate's MBB. + unsigned Flags = 0x0; + + /// True if initLRU has been called on this Candidate. + bool LRUWasSet = false; + /// Return the number of instructions in this Candidate. unsigned getLength() const { return Len; } @@ -99,9 +102,7 @@ public: } /// Returns the call overhead of this candidate if it is in the list. - unsigned getCallOverhead() const { - return InCandidateList ? CallOverhead : 0; - } + unsigned getCallOverhead() const { return CallOverhead; } MachineBasicBlock::iterator &front() { return FirstInst; } MachineBasicBlock::iterator &back() { return LastInst; } @@ -120,9 +121,9 @@ public: Candidate(unsigned StartIdx, unsigned Len, MachineBasicBlock::iterator &FirstInst, MachineBasicBlock::iterator &LastInst, MachineBasicBlock *MBB, - unsigned FunctionIdx) + unsigned FunctionIdx, unsigned Flags) : StartIdx(StartIdx), Len(Len), FirstInst(FirstInst), LastInst(LastInst), - MBB(MBB), FunctionIdx(FunctionIdx) {} + MBB(MBB), FunctionIdx(FunctionIdx), Flags(Flags) {} Candidate() {} /// Used to ensure that \p Candidates are outlined in an order that @@ -138,6 +139,10 @@ public: void initLRU(const TargetRegisterInfo &TRI) { assert(MBB->getParent()->getRegInfo().tracksLiveness() && "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (LRUWasSet) + return; + LRUWasSet = true; LRU.init(TRI); LRU.addLiveOuts(*MBB); @@ -158,24 +163,13 @@ public: /// class of candidate. struct OutlinedFunction { -private: - /// The number of candidates for this \p OutlinedFunction. - unsigned OccurrenceCount = 0; - public: - std::vector> Candidates; + std::vector Candidates; /// The actual outlined function created. /// This is initialized after we go through and create the actual function. MachineFunction *MF = nullptr; - /// A number assigned to this function which appears at the end of its name. - unsigned Name; - - /// The sequence of integers corresponding to the instructions in this - /// function. - std::vector Sequence; - /// Represents the size of a sequence in bytes. (Some instructions vary /// widely in size, so just counting the instructions isn't very useful.) unsigned SequenceSize; @@ -187,49 +181,41 @@ public: unsigned FrameConstructionID; /// Return the number of candidates for this \p OutlinedFunction. - unsigned getOccurrenceCount() { return OccurrenceCount; } - - /// Decrement the occurrence count of this OutlinedFunction and return the - /// new count. - unsigned decrement() { - assert(OccurrenceCount > 0 && "Can't decrement an empty function!"); - OccurrenceCount--; - return getOccurrenceCount(); - } + unsigned getOccurrenceCount() const { return Candidates.size(); } /// Return the number of bytes it would take to outline this /// function. - unsigned getOutliningCost() { + unsigned getOutliningCost() const { unsigned CallOverhead = 0; - for (std::shared_ptr &C : Candidates) - CallOverhead += C->getCallOverhead(); + for (const Candidate &C : Candidates) + CallOverhead += C.getCallOverhead(); return CallOverhead + SequenceSize + FrameOverhead; } /// Return the size in bytes of the unoutlined sequences. - unsigned getNotOutlinedCost() { return OccurrenceCount * SequenceSize; } + unsigned getNotOutlinedCost() const { + return getOccurrenceCount() * SequenceSize; + } /// Return the number of instructions that would be saved by outlining /// this function. - unsigned getBenefit() { + unsigned getBenefit() const { unsigned NotOutlinedCost = getNotOutlinedCost(); unsigned OutlinedCost = getOutliningCost(); return (NotOutlinedCost < OutlinedCost) ? 0 : NotOutlinedCost - OutlinedCost; } - OutlinedFunction(std::vector &Cands, - unsigned SequenceSize, unsigned FrameOverhead, - unsigned FrameConstructionID) - : SequenceSize(SequenceSize), FrameOverhead(FrameOverhead), - FrameConstructionID(FrameConstructionID) { - OccurrenceCount = Cands.size(); - for (Candidate &C : Cands) - Candidates.push_back(std::make_shared(C)); - - unsigned B = getBenefit(); - for (std::shared_ptr &C : Candidates) - C->Benefit = B; + /// Return the number of instructions in this sequence. + unsigned getNumInstrs() const { return Candidates[0].getLength(); } + + OutlinedFunction(std::vector &Candidates, unsigned SequenceSize, + unsigned FrameOverhead, unsigned FrameConstructionID) + : Candidates(Candidates), SequenceSize(SequenceSize), + FrameOverhead(FrameOverhead), FrameConstructionID(FrameConstructionID) { + const unsigned B = getBenefit(); + for (Candidate &C : Candidates) + C.Benefit = B; } OutlinedFunction() {} diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h index 3aba0bba7d1a..a031c92d914f 100644 --- a/include/llvm/CodeGen/MachinePassRegistry.h +++ b/include/llvm/CodeGen/MachinePassRegistry.h @@ -24,22 +24,20 @@ namespace llvm { -using MachinePassCtor = void *(*)(); - //===----------------------------------------------------------------------===// /// /// MachinePassRegistryListener - Listener to adds and removals of nodes in /// registration list. /// //===----------------------------------------------------------------------===// -class MachinePassRegistryListener { - virtual void anchor(); +template class MachinePassRegistryListener { + virtual void anchor() {} public: MachinePassRegistryListener() = default; virtual ~MachinePassRegistryListener() = default; - virtual void NotifyAdd(StringRef N, MachinePassCtor C, StringRef D) = 0; + virtual void NotifyAdd(StringRef N, PassCtorTy C, StringRef D) = 0; virtual void NotifyRemove(StringRef N) = 0; }; @@ -48,15 +46,15 @@ public: /// MachinePassRegistryNode - Machine pass node stored in registration list. /// //===----------------------------------------------------------------------===// -class MachinePassRegistryNode { +template class MachinePassRegistryNode { private: MachinePassRegistryNode *Next = nullptr; // Next function pass in list. StringRef Name; // Name of function pass. StringRef Description; // Description string. - MachinePassCtor Ctor; // Function pass creator. + PassCtorTy Ctor; // Pass creator. public: - MachinePassRegistryNode(const char *N, const char *D, MachinePassCtor C) + MachinePassRegistryNode(const char *N, const char *D, PassCtorTy C) : Name(N), Description(D), Ctor(C) {} // Accessors @@ -64,7 +62,7 @@ public: MachinePassRegistryNode **getNextAddress() { return &Next; } StringRef getName() const { return Name; } StringRef getDescription() const { return Description; } - MachinePassCtor getCtor() const { return Ctor; } + PassCtorTy getCtor() const { return Ctor; } void setNext(MachinePassRegistryNode *N) { Next = N; } }; @@ -73,11 +71,12 @@ public: /// MachinePassRegistry - Track the registration of machine passes. /// //===----------------------------------------------------------------------===// -class MachinePassRegistry { +template class MachinePassRegistry { private: - MachinePassRegistryNode *List; // List of registry nodes. - MachinePassCtor Default; // Default function pass creator. - MachinePassRegistryListener *Listener; // Listener for list adds are removes. + MachinePassRegistryNode *List; // List of registry nodes. + PassCtorTy Default; // Default function pass creator. + MachinePassRegistryListener + *Listener; // Listener for list adds are removes. public: // NO CONSTRUCTOR - we don't want static constructor ordering to mess @@ -85,19 +84,47 @@ public: // Accessors. // - MachinePassRegistryNode *getList() { return List; } - MachinePassCtor getDefault() { return Default; } - void setDefault(MachinePassCtor C) { Default = C; } - void setDefault(StringRef Name); - void setListener(MachinePassRegistryListener *L) { Listener = L; } + MachinePassRegistryNode *getList() { return List; } + PassCtorTy getDefault() { return Default; } + void setDefault(PassCtorTy C) { Default = C; } + /// setDefault - Set the default constructor by name. + void setDefault(StringRef Name) { + PassCtorTy Ctor = nullptr; + for (MachinePassRegistryNode *R = getList(); R; + R = R->getNext()) { + if (R->getName() == Name) { + Ctor = R->getCtor(); + break; + } + } + assert(Ctor && "Unregistered pass name"); + setDefault(Ctor); + } + void setListener(MachinePassRegistryListener *L) { Listener = L; } /// Add - Adds a function pass to the registration list. /// - void Add(MachinePassRegistryNode *Node); + void Add(MachinePassRegistryNode *Node) { + Node->setNext(List); + List = Node; + if (Listener) + Listener->NotifyAdd(Node->getName(), Node->getCtor(), + Node->getDescription()); + } /// Remove - Removes a function pass from the registration list. /// - void Remove(MachinePassRegistryNode *Node); + void Remove(MachinePassRegistryNode *Node) { + for (MachinePassRegistryNode **I = &List; *I; + I = (*I)->getNextAddress()) { + if (*I == Node) { + if (Listener) + Listener->NotifyRemove(Node->getName()); + *I = (*I)->getNext(); + break; + } + } + } }; //===----------------------------------------------------------------------===// @@ -105,9 +132,11 @@ public: /// RegisterPassParser class - Handle the addition of new machine passes. /// //===----------------------------------------------------------------------===// -template -class RegisterPassParser : public MachinePassRegistryListener, - public cl::parser { +template +class RegisterPassParser + : public MachinePassRegistryListener< + typename RegistryClass::FunctionPassCtor>, + public cl::parser { public: RegisterPassParser(cl::Option &O) : cl::parser(O) {} @@ -129,8 +158,9 @@ public: } // Implement the MachinePassRegistryListener callbacks. - void NotifyAdd(StringRef N, MachinePassCtor C, StringRef D) override { - this->addLiteralOption(N, (typename RegistryClass::FunctionPassCtor)C, D); + void NotifyAdd(StringRef N, typename RegistryClass::FunctionPassCtor C, + StringRef D) override { + this->addLiteralOption(N, C, D); } void NotifyRemove(StringRef N) override { this->removeLiteralOption(N); diff --git a/include/llvm/CodeGen/MachinePipeliner.h b/include/llvm/CodeGen/MachinePipeliner.h new file mode 100644 index 000000000000..38cb33e90e63 --- /dev/null +++ b/include/llvm/CodeGen/MachinePipeliner.h @@ -0,0 +1,608 @@ +//===- MachinePipeliner.h - Machine Software Pipeliner Pass -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// An implementation of the Swing Modulo Scheduling (SMS) software pipeliner. +// +// Software pipelining (SWP) is an instruction scheduling technique for loops +// that overlap loop iterations and exploits ILP via a compiler transformation. +// +// Swing Modulo Scheduling is an implementation of software pipelining +// that generates schedules that are near optimal in terms of initiation +// interval, register requirements, and stage count. See the papers: +// +// "Swing Modulo Scheduling: A Lifetime-Sensitive Approach", by J. Llosa, +// A. Gonzalez, E. Ayguade, and M. Valero. In PACT '96 Proceedings of the 1996 +// Conference on Parallel Architectures and Compilation Techiniques. +// +// "Lifetime-Sensitive Modulo Scheduling in a Production Environment", by J. +// Llosa, E. Ayguade, A. Gonzalez, M. Valero, and J. Eckhardt. In IEEE +// Transactions on Computers, Vol. 50, No. 3, 2001. +// +// "An Implementation of Swing Modulo Scheduling With Extensions for +// Superblocks", by T. Lattner, Master's Thesis, University of Illinois at +// Urbana-Champaign, 2005. +// +// +// The SMS algorithm consists of three main steps after computing the minimal +// initiation interval (MII). +// 1) Analyze the dependence graph and compute information about each +// instruction in the graph. +// 2) Order the nodes (instructions) by priority based upon the heuristics +// described in the algorithm. +// 3) Attempt to schedule the nodes in the specified order using the MII. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIB_CODEGEN_MACHINEPIPELINER_H +#define LLVM_LIB_CODEGEN_MACHINEPIPELINER_H + +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/CodeGen/TargetInstrInfo.h" + +namespace llvm { + +class NodeSet; +class SMSchedule; + +extern cl::opt SwpEnableCopyToPhi; + +/// The main class in the implementation of the target independent +/// software pipeliner pass. +class MachinePipeliner : public MachineFunctionPass { +public: + MachineFunction *MF = nullptr; + const MachineLoopInfo *MLI = nullptr; + const MachineDominatorTree *MDT = nullptr; + const InstrItineraryData *InstrItins; + const TargetInstrInfo *TII = nullptr; + RegisterClassInfo RegClassInfo; + +#ifndef NDEBUG + static int NumTries; +#endif + + /// Cache the target analysis information about the loop. + struct LoopInfo { + MachineBasicBlock *TBB = nullptr; + MachineBasicBlock *FBB = nullptr; + SmallVector BrCond; + MachineInstr *LoopInductionVar = nullptr; + MachineInstr *LoopCompare = nullptr; + }; + LoopInfo LI; + + static char ID; + + MachinePipeliner() : MachineFunctionPass(ID) { + initializeMachinePipelinerPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + void preprocessPhiNodes(MachineBasicBlock &B); + bool canPipelineLoop(MachineLoop &L); + bool scheduleLoop(MachineLoop &L); + bool swingModuloScheduler(MachineLoop &L); +}; + +/// This class builds the dependence graph for the instructions in a loop, +/// and attempts to schedule the instructions using the SMS algorithm. +class SwingSchedulerDAG : public ScheduleDAGInstrs { + MachinePipeliner &Pass; + /// The minimum initiation interval between iterations for this schedule. + unsigned MII = 0; + /// Set to true if a valid pipelined schedule is found for the loop. + bool Scheduled = false; + MachineLoop &Loop; + LiveIntervals &LIS; + const RegisterClassInfo &RegClassInfo; + + /// A toplogical ordering of the SUnits, which is needed for changing + /// dependences and iterating over the SUnits. + ScheduleDAGTopologicalSort Topo; + + struct NodeInfo { + int ASAP = 0; + int ALAP = 0; + int ZeroLatencyDepth = 0; + int ZeroLatencyHeight = 0; + + NodeInfo() = default; + }; + /// Computed properties for each node in the graph. + std::vector ScheduleInfo; + + enum OrderKind { BottomUp = 0, TopDown = 1 }; + /// Computed node ordering for scheduling. + SetVector NodeOrder; + + using NodeSetType = SmallVector; + using ValueMapTy = DenseMap; + using MBBVectorTy = SmallVectorImpl; + using InstrMapTy = DenseMap; + + /// Instructions to change when emitting the final schedule. + DenseMap> InstrChanges; + + /// We may create a new instruction, so remember it because it + /// must be deleted when the pass is finished. + SmallPtrSet NewMIs; + + /// Ordered list of DAG postprocessing steps. + std::vector> Mutations; + + /// Helper class to implement Johnson's circuit finding algorithm. + class Circuits { + std::vector &SUnits; + SetVector Stack; + BitVector Blocked; + SmallVector, 10> B; + SmallVector, 16> AdjK; + // Node to Index from ScheduleDAGTopologicalSort + std::vector *Node2Idx; + unsigned NumPaths; + static unsigned MaxPaths; + + public: + Circuits(std::vector &SUs, ScheduleDAGTopologicalSort &Topo) + : SUnits(SUs), Blocked(SUs.size()), B(SUs.size()), AdjK(SUs.size()) { + Node2Idx = new std::vector(SUs.size()); + unsigned Idx = 0; + for (const auto &NodeNum : Topo) + Node2Idx->at(NodeNum) = Idx++; + } + + ~Circuits() { delete Node2Idx; } + + /// Reset the data structures used in the circuit algorithm. + void reset() { + Stack.clear(); + Blocked.reset(); + B.assign(SUnits.size(), SmallPtrSet()); + NumPaths = 0; + } + + void createAdjacencyStructure(SwingSchedulerDAG *DAG); + bool circuit(int V, int S, NodeSetType &NodeSets, bool HasBackedge = false); + void unblock(int U); + }; + + struct CopyToPhiMutation : public ScheduleDAGMutation { + void apply(ScheduleDAGInstrs *DAG) override; + }; + +public: + SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis, + const RegisterClassInfo &rci) + : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis), + RegClassInfo(rci), Topo(SUnits, &ExitSU) { + P.MF->getSubtarget().getSMSMutations(Mutations); + if (SwpEnableCopyToPhi) + Mutations.push_back(llvm::make_unique()); + } + + void schedule() override; + void finishBlock() override; + + /// Return true if the loop kernel has been scheduled. + bool hasNewSchedule() { return Scheduled; } + + /// Return the earliest time an instruction may be scheduled. + int getASAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ASAP; } + + /// Return the latest time an instruction my be scheduled. + int getALAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ALAP; } + + /// The mobility function, which the number of slots in which + /// an instruction may be scheduled. + int getMOV(SUnit *Node) { return getALAP(Node) - getASAP(Node); } + + /// The depth, in the dependence graph, for a node. + unsigned getDepth(SUnit *Node) { return Node->getDepth(); } + + /// The maximum unweighted length of a path from an arbitrary node to the + /// given node in which each edge has latency 0 + int getZeroLatencyDepth(SUnit *Node) { + return ScheduleInfo[Node->NodeNum].ZeroLatencyDepth; + } + + /// The height, in the dependence graph, for a node. + unsigned getHeight(SUnit *Node) { return Node->getHeight(); } + + /// The maximum unweighted length of a path from the given node to an + /// arbitrary node in which each edge has latency 0 + int getZeroLatencyHeight(SUnit *Node) { + return ScheduleInfo[Node->NodeNum].ZeroLatencyHeight; + } + + /// Return true if the dependence is a back-edge in the data dependence graph. + /// Since the DAG doesn't contain cycles, we represent a cycle in the graph + /// using an anti dependence from a Phi to an instruction. + bool isBackedge(SUnit *Source, const SDep &Dep) { + if (Dep.getKind() != SDep::Anti) + return false; + return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI(); + } + + bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc = true); + + /// The distance function, which indicates that operation V of iteration I + /// depends on operations U of iteration I-distance. + unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep) { + // Instructions that feed a Phi have a distance of 1. Computing larger + // values for arrays requires data dependence information. + if (V->getInstr()->isPHI() && Dep.getKind() == SDep::Anti) + return 1; + return 0; + } + + /// Set the Minimum Initiation Interval for this schedule attempt. + void setMII(unsigned mii) { MII = mii; } + + void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule); + + void fixupRegisterOverlaps(std::deque &Instrs); + + /// Return the new base register that was stored away for the changed + /// instruction. + unsigned getInstrBaseReg(SUnit *SU) { + DenseMap>::iterator It = + InstrChanges.find(SU); + if (It != InstrChanges.end()) + return It->second.first; + return 0; + } + + void addMutation(std::unique_ptr Mutation) { + Mutations.push_back(std::move(Mutation)); + } + + static bool classof(const ScheduleDAGInstrs *DAG) { return true; } + +private: + void addLoopCarriedDependences(AliasAnalysis *AA); + void updatePhiDependences(); + void changeDependences(); + unsigned calculateResMII(); + unsigned calculateRecMII(NodeSetType &RecNodeSets); + void findCircuits(NodeSetType &NodeSets); + void fuseRecs(NodeSetType &NodeSets); + void removeDuplicateNodes(NodeSetType &NodeSets); + void computeNodeFunctions(NodeSetType &NodeSets); + void registerPressureFilter(NodeSetType &NodeSets); + void colocateNodeSets(NodeSetType &NodeSets); + void checkNodeSets(NodeSetType &NodeSets); + void groupRemainingNodes(NodeSetType &NodeSets); + void addConnectedNodes(SUnit *SU, NodeSet &NewSet, + SetVector &NodesAdded); + void computeNodeOrder(NodeSetType &NodeSets); + void checkValidNodeOrder(const NodeSetType &Circuits) const; + bool schedulePipeline(SMSchedule &Schedule); + void generatePipelinedLoop(SMSchedule &Schedule); + void generateProlog(SMSchedule &Schedule, unsigned LastStage, + MachineBasicBlock *KernelBB, ValueMapTy *VRMap, + MBBVectorTy &PrologBBs); + void generateEpilog(SMSchedule &Schedule, unsigned LastStage, + MachineBasicBlock *KernelBB, ValueMapTy *VRMap, + MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs); + void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap, unsigned LastStageNum, + unsigned CurStageNum, bool IsLast); + void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap, unsigned LastStageNum, + unsigned CurStageNum, bool IsLast); + void removeDeadInstructions(MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs); + void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, + SMSchedule &Schedule); + void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs, SMSchedule &Schedule, + ValueMapTy *VRMap); + bool computeDelta(MachineInstr &MI, unsigned &Delta); + void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI, + unsigned Num); + MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum); + MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum, + SMSchedule &Schedule); + void updateInstruction(MachineInstr *NewMI, bool LastDef, + unsigned CurStageNum, unsigned InstrStageNum, + SMSchedule &Schedule, ValueMapTy *VRMap); + MachineInstr *findDefInLoop(unsigned Reg); + unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal, + unsigned LoopStage, ValueMapTy *VRMap, + MachineBasicBlock *BB); + void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap); + void rewriteScheduledInstr(MachineBasicBlock *BB, SMSchedule &Schedule, + InstrMapTy &InstrMap, unsigned CurStageNum, + unsigned PhiNum, MachineInstr *Phi, + unsigned OldReg, unsigned NewReg, + unsigned PrevReg = 0); + bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos, + unsigned &OffsetPos, unsigned &NewBase, + int64_t &NewOffset); + void postprocessDAG(); +}; + +/// A NodeSet contains a set of SUnit DAG nodes with additional information +/// that assigns a priority to the set. +class NodeSet { + SetVector Nodes; + bool HasRecurrence = false; + unsigned RecMII = 0; + int MaxMOV = 0; + unsigned MaxDepth = 0; + unsigned Colocate = 0; + SUnit *ExceedPressure = nullptr; + unsigned Latency = 0; + +public: + using iterator = SetVector::const_iterator; + + NodeSet() = default; + NodeSet(iterator S, iterator E) : Nodes(S, E), HasRecurrence(true) { + Latency = 0; + for (unsigned i = 0, e = Nodes.size(); i < e; ++i) + for (const SDep &Succ : Nodes[i]->Succs) + if (Nodes.count(Succ.getSUnit())) + Latency += Succ.getLatency(); + } + + bool insert(SUnit *SU) { return Nodes.insert(SU); } + + void insert(iterator S, iterator E) { Nodes.insert(S, E); } + + template bool remove_if(UnaryPredicate P) { + return Nodes.remove_if(P); + } + + unsigned count(SUnit *SU) const { return Nodes.count(SU); } + + bool hasRecurrence() { return HasRecurrence; }; + + unsigned size() const { return Nodes.size(); } + + bool empty() const { return Nodes.empty(); } + + SUnit *getNode(unsigned i) const { return Nodes[i]; }; + + void setRecMII(unsigned mii) { RecMII = mii; }; + + void setColocate(unsigned c) { Colocate = c; }; + + void setExceedPressure(SUnit *SU) { ExceedPressure = SU; } + + bool isExceedSU(SUnit *SU) { return ExceedPressure == SU; } + + int compareRecMII(NodeSet &RHS) { return RecMII - RHS.RecMII; } + + int getRecMII() { return RecMII; } + + /// Summarize node functions for the entire node set. + void computeNodeSetInfo(SwingSchedulerDAG *SSD) { + for (SUnit *SU : *this) { + MaxMOV = std::max(MaxMOV, SSD->getMOV(SU)); + MaxDepth = std::max(MaxDepth, SSD->getDepth(SU)); + } + } + + unsigned getLatency() { return Latency; } + + unsigned getMaxDepth() { return MaxDepth; } + + void clear() { + Nodes.clear(); + RecMII = 0; + HasRecurrence = false; + MaxMOV = 0; + MaxDepth = 0; + Colocate = 0; + ExceedPressure = nullptr; + } + + operator SetVector &() { return Nodes; } + + /// Sort the node sets by importance. First, rank them by recurrence MII, + /// then by mobility (least mobile done first), and finally by depth. + /// Each node set may contain a colocate value which is used as the first + /// tie breaker, if it's set. + bool operator>(const NodeSet &RHS) const { + if (RecMII == RHS.RecMII) { + if (Colocate != 0 && RHS.Colocate != 0 && Colocate != RHS.Colocate) + return Colocate < RHS.Colocate; + if (MaxMOV == RHS.MaxMOV) + return MaxDepth > RHS.MaxDepth; + return MaxMOV < RHS.MaxMOV; + } + return RecMII > RHS.RecMII; + } + + bool operator==(const NodeSet &RHS) const { + return RecMII == RHS.RecMII && MaxMOV == RHS.MaxMOV && + MaxDepth == RHS.MaxDepth; + } + + bool operator!=(const NodeSet &RHS) const { return !operator==(RHS); } + + iterator begin() { return Nodes.begin(); } + iterator end() { return Nodes.end(); } + void print(raw_ostream &os) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif +}; + +/// This class represents the scheduled code. The main data structure is a +/// map from scheduled cycle to instructions. During scheduling, the +/// data structure explicitly represents all stages/iterations. When +/// the algorithm finshes, the schedule is collapsed into a single stage, +/// which represents instructions from different loop iterations. +/// +/// The SMS algorithm allows negative values for cycles, so the first cycle +/// in the schedule is the smallest cycle value. +class SMSchedule { +private: + /// Map from execution cycle to instructions. + DenseMap> ScheduledInstrs; + + /// Map from instruction to execution cycle. + std::map InstrToCycle; + + /// Map for each register and the max difference between its uses and def. + /// The first element in the pair is the max difference in stages. The + /// second is true if the register defines a Phi value and loop value is + /// scheduled before the Phi. + std::map> RegToStageDiff; + + /// Keep track of the first cycle value in the schedule. It starts + /// as zero, but the algorithm allows negative values. + int FirstCycle = 0; + + /// Keep track of the last cycle value in the schedule. + int LastCycle = 0; + + /// The initiation interval (II) for the schedule. + int InitiationInterval = 0; + + /// Target machine information. + const TargetSubtargetInfo &ST; + + /// Virtual register information. + MachineRegisterInfo &MRI; + + std::unique_ptr Resources; + +public: + SMSchedule(MachineFunction *mf) + : ST(mf->getSubtarget()), MRI(mf->getRegInfo()), + Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {} + + void reset() { + ScheduledInstrs.clear(); + InstrToCycle.clear(); + RegToStageDiff.clear(); + FirstCycle = 0; + LastCycle = 0; + InitiationInterval = 0; + } + + /// Set the initiation interval for this schedule. + void setInitiationInterval(int ii) { InitiationInterval = ii; } + + /// Return the first cycle in the completed schedule. This + /// can be a negative value. + int getFirstCycle() const { return FirstCycle; } + + /// Return the last cycle in the finalized schedule. + int getFinalCycle() const { return FirstCycle + InitiationInterval - 1; } + + /// Return the cycle of the earliest scheduled instruction in the dependence + /// chain. + int earliestCycleInChain(const SDep &Dep); + + /// Return the cycle of the latest scheduled instruction in the dependence + /// chain. + int latestCycleInChain(const SDep &Dep); + + void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart, + int *MinEnd, int *MaxStart, int II, SwingSchedulerDAG *DAG); + bool insert(SUnit *SU, int StartCycle, int EndCycle, int II); + + /// Iterators for the cycle to instruction map. + using sched_iterator = DenseMap>::iterator; + using const_sched_iterator = + DenseMap>::const_iterator; + + /// Return true if the instruction is scheduled at the specified stage. + bool isScheduledAtStage(SUnit *SU, unsigned StageNum) { + return (stageScheduled(SU) == (int)StageNum); + } + + /// Return the stage for a scheduled instruction. Return -1 if + /// the instruction has not been scheduled. + int stageScheduled(SUnit *SU) const { + std::map::const_iterator it = InstrToCycle.find(SU); + if (it == InstrToCycle.end()) + return -1; + return (it->second - FirstCycle) / InitiationInterval; + } + + /// Return the cycle for a scheduled instruction. This function normalizes + /// the first cycle to be 0. + unsigned cycleScheduled(SUnit *SU) const { + std::map::const_iterator it = InstrToCycle.find(SU); + assert(it != InstrToCycle.end() && "Instruction hasn't been scheduled."); + return (it->second - FirstCycle) % InitiationInterval; + } + + /// Return the maximum stage count needed for this schedule. + unsigned getMaxStageCount() { + return (LastCycle - FirstCycle) / InitiationInterval; + } + + /// Return the max. number of stages/iterations that can occur between a + /// register definition and its uses. + unsigned getStagesForReg(int Reg, unsigned CurStage) { + std::pair Stages = RegToStageDiff[Reg]; + if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second) + return 1; + return Stages.first; + } + + /// The number of stages for a Phi is a little different than other + /// instructions. The minimum value computed in RegToStageDiff is 1 + /// because we assume the Phi is needed for at least 1 iteration. + /// This is not the case if the loop value is scheduled prior to the + /// Phi in the same stage. This function returns the number of stages + /// or iterations needed between the Phi definition and any uses. + unsigned getStagesForPhi(int Reg) { + std::pair Stages = RegToStageDiff[Reg]; + if (Stages.second) + return Stages.first; + return Stages.first - 1; + } + + /// Return the instructions that are scheduled at the specified cycle. + std::deque &getInstructions(int cycle) { + return ScheduledInstrs[cycle]; + } + + bool isValidSchedule(SwingSchedulerDAG *SSD); + void finalizeSchedule(SwingSchedulerDAG *SSD); + void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU, + std::deque &Insts); + bool isLoopCarried(SwingSchedulerDAG *SSD, MachineInstr &Phi); + bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD, MachineInstr *Def, + MachineOperand &MO); + void print(raw_ostream &os) const; + void dump() const; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_CODEGEN_MACHINEPIPELINER_H diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index 5bf4a49c8b3b..fef010a23ef9 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -689,15 +689,14 @@ public: unsigned MinNumRegs = 0); /// Constrain the register class or the register bank of the virtual register - /// \p Reg to be a common subclass and a common bank of both registers - /// provided respectively. Do nothing if any of the attributes (classes, - /// banks, or low-level types) of the registers are deemed incompatible, or if - /// the resulting register will have a class smaller than before and of size - /// less than \p MinNumRegs. Return true if such register attributes exist, - /// false otherwise. + /// \p Reg (and low-level type) to be a common subclass or a common bank of + /// both registers provided respectively (and a common low-level type). Do + /// nothing if any of the attributes (classes, banks, or low-level types) of + /// the registers are deemed incompatible, or if the resulting register will + /// have a class smaller than before and of size less than \p MinNumRegs. + /// Return true if such register attributes exist, false otherwise. /// - /// \note Assumes that each register has either a low-level type or a class - /// assigned, but not both. Use this method instead of constrainRegClass and + /// \note Use this method instead of constrainRegClass and /// RegisterBankInfo::constrainGenericRegister everywhere but SelectionDAG /// ISel / FastISel and GlobalISel's InstructionSelect pass respectively. bool constrainRegAttrs(unsigned Reg, unsigned ConstrainingReg, @@ -717,6 +716,10 @@ public: unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name = ""); + /// Create and return a new virtual register in the function with the same + /// attributes as the given register. + unsigned cloneVirtualRegister(unsigned VReg, StringRef Name = ""); + /// Get the low-level type of \p Reg or LLT{} if Reg is not a generic /// (target independent) virtual register. LLT getType(unsigned Reg) const { diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 85ffa4eda2b8..4bc31ae7c61a 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -132,17 +132,19 @@ struct MachineSchedContext { /// MachineSchedRegistry provides a selection of available machine instruction /// schedulers. -class MachineSchedRegistry : public MachinePassRegistryNode { +class MachineSchedRegistry + : public MachinePassRegistryNode< + ScheduleDAGInstrs *(*)(MachineSchedContext *)> { public: using ScheduleDAGCtor = ScheduleDAGInstrs *(*)(MachineSchedContext *); // RegisterPassParser requires a (misnamed) FunctionPassCtor type. using FunctionPassCtor = ScheduleDAGCtor; - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; MachineSchedRegistry(const char *N, const char *D, ScheduleDAGCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) { + : MachinePassRegistryNode(N, D, C) { Registry.Add(this); } @@ -158,7 +160,7 @@ public: return (MachineSchedRegistry *)Registry.getList(); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; @@ -466,6 +468,9 @@ public: PressureDiff &getPressureDiff(const SUnit *SU) { return SUPressureDiffs[SU->NodeNum]; } + const PressureDiff &getPressureDiff(const SUnit *SU) const { + return SUPressureDiffs[SU->NodeNum]; + } /// Compute a DFSResult after DAG building is complete, and before any /// queue comparisons. @@ -491,6 +496,8 @@ public: /// Compute the cyclic critical path through the DAG. unsigned computeCyclicCriticalPath(); + void dump() const override; + protected: // Top-Level entry points for the schedule() driver... @@ -787,7 +794,7 @@ public: /// Represent the type of SchedCandidate found within a single queue. /// pickNodeBidirectional depends on these listed by decreasing priority. enum CandReason : uint8_t { - NoCand, Only1, PhysRegCopy, RegExcess, RegCritical, Stall, Cluster, Weak, + NoCand, Only1, PhysReg, RegExcess, RegCritical, Stall, Cluster, Weak, RegMax, ResourceReduce, ResourceDemand, BotHeightReduce, BotPathReduce, TopDepthReduce, TopPathReduce, NextDefUse, NodeOrder}; @@ -895,6 +902,10 @@ protected: #ifndef NDEBUG void traceCandidate(const SchedCandidate &Cand); #endif + +private: + bool shouldReduceLatency(const CandPolicy &Policy, SchedBoundary &CurrZone, + bool ComputeRemLatency, unsigned &RemLatency) const; }; // Utility functions used by heuristics in tryCandidate(). @@ -917,7 +928,7 @@ bool tryPressure(const PressureChange &TryP, const TargetRegisterInfo *TRI, const MachineFunction &MF); unsigned getWeakLeft(const SUnit *SU, bool isTop); -int biasPhysRegCopy(const SUnit *SU, bool isTop); +int biasPhysReg(const SUnit *SU, bool isTop); /// GenericScheduler shrinks the unscheduled zone using heuristics to balance /// the schedule. @@ -995,7 +1006,7 @@ protected: const RegPressureTracker &RPTracker, SchedCandidate &Candidate); - void reschedulePhysRegCopies(SUnit *SU, bool isTop); + void reschedulePhysReg(SUnit *SU, bool isTop); }; /// PostGenericScheduler - Interface to the scheduling algorithm used by diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index cb12b14f4435..acf1ebb5bc83 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -379,14 +379,20 @@ namespace llvm { /// FunctionPass *createInterleavedAccessPass(); + /// InterleavedLoadCombines Pass - This pass identifies interleaved loads and + /// combines them into wide loads detectable by InterleavedAccessPass + /// + FunctionPass *createInterleavedLoadCombinePass(); + /// LowerEmuTLS - This pass generates __emutls_[vt].xyz variables for all /// TLS variables for the emulated TLS model. /// ModulePass *createLowerEmuTLSPass(); - /// This pass lowers the \@llvm.load.relative intrinsic to instructions. - /// This is unsafe to do earlier because a pass may combine the constant - /// initializer into the load, which may result in an overflowing evaluation. + /// This pass lowers the \@llvm.load.relative and \@llvm.objc.* intrinsics to + /// instructions. This is unsafe to do earlier because a pass may combine the + /// constant initializer into the load, which may result in an overflowing + /// evaluation. ModulePass *createPreISelIntrinsicLoweringPass(); /// GlobalMerge - This pass merges internal (by default) globals into structs diff --git a/include/llvm/CodeGen/PreISelIntrinsicLowering.h b/include/llvm/CodeGen/PreISelIntrinsicLowering.h index 7a007eb8bcea..b7f83e515b7e 100644 --- a/include/llvm/CodeGen/PreISelIntrinsicLowering.h +++ b/include/llvm/CodeGen/PreISelIntrinsicLowering.h @@ -7,7 +7,8 @@ // //===----------------------------------------------------------------------===// // -// This pass implements IR lowering for the llvm.load.relative intrinsic. +// This pass implements IR lowering for the llvm.load.relative and llvm.objc.* +// intrinsics. // //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_PREISELINTRINSICLOWERING_H diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h index bdf0bb731540..f66191bc9fb4 100644 --- a/include/llvm/CodeGen/PseudoSourceValue.h +++ b/include/llvm/CodeGen/PseudoSourceValue.h @@ -36,7 +36,7 @@ raw_ostream &operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); /// below the stack frame (e.g., argument space), or constant pool. class PseudoSourceValue { public: - enum PSVKind { + enum PSVKind : unsigned { Stack, GOT, JumpTable, @@ -48,7 +48,7 @@ public: }; private: - PSVKind Kind; + unsigned Kind; unsigned AddressSpace; friend raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -60,11 +60,11 @@ private: virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); + explicit PseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); virtual ~PseudoSourceValue(); - PSVKind kind() const { return Kind; } + unsigned kind() const { return Kind; } bool isStack() const { return Kind == Stack; } bool isGOT() const { return Kind == GOT; } @@ -116,7 +116,7 @@ public: class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); + CallEntryPseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); public: bool isConstant(const MachineFrameInfo *) const override; diff --git a/include/llvm/CodeGen/RegAllocRegistry.h b/include/llvm/CodeGen/RegAllocRegistry.h index 481747dc163e..b518fbb9c9da 100644 --- a/include/llvm/CodeGen/RegAllocRegistry.h +++ b/include/llvm/CodeGen/RegAllocRegistry.h @@ -26,14 +26,14 @@ class FunctionPass; /// RegisterRegAlloc class - Track the registration of register allocators. /// //===----------------------------------------------------------------------===// -class RegisterRegAlloc : public MachinePassRegistryNode { +class RegisterRegAlloc : public MachinePassRegistryNode { public: using FunctionPassCtor = FunctionPass *(*)(); - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) { + : MachinePassRegistryNode(N, D, C) { Registry.Add(this); } @@ -48,15 +48,11 @@ public: return (RegisterRegAlloc *)Registry.getList(); } - static FunctionPassCtor getDefault() { - return (FunctionPassCtor)Registry.getDefault(); - } + static FunctionPassCtor getDefault() { return Registry.getDefault(); } - static void setDefault(FunctionPassCtor C) { - Registry.setDefault((MachinePassCtor)C); - } + static void setDefault(FunctionPassCtor C) { Registry.setDefault(C); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; diff --git a/include/llvm/CodeGen/RegisterUsageInfo.h b/include/llvm/CodeGen/RegisterUsageInfo.h index efd175eeed30..efecc61d9c30 100644 --- a/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/include/llvm/CodeGen/RegisterUsageInfo.h @@ -29,7 +29,7 @@ namespace llvm { class Function; -class TargetMachine; +class LLVMTargetMachine; class PhysicalRegisterUsageInfo : public ImmutablePass { public: @@ -41,7 +41,7 @@ public: } /// Set TargetMachine which is used to print analysis. - void setTargetMachine(const TargetMachine &TM); + void setTargetMachine(const LLVMTargetMachine &TM); bool doInitialization(Module &M) override; @@ -63,7 +63,7 @@ private: /// and 1 means content of register will be preserved around function call. DenseMap> RegMasks; - const TargetMachine *TM; + const LLVMTargetMachine *TM; }; } // end namespace llvm diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 56adc2e2fbfa..0870d67db390 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -33,15 +33,15 @@ namespace llvm { template class GraphWriter; +class LLVMTargetMachine; class MachineFunction; class MachineRegisterInfo; class MCInstrDesc; struct MCSchedClassDesc; -class ScheduleDAG; class SDNode; class SUnit; +class ScheduleDAG; class TargetInstrInfo; -class TargetMachine; class TargetRegisterClass; class TargetRegisterInfo; @@ -236,8 +236,7 @@ class TargetRegisterInfo; Contents.Reg = Reg; } - raw_ostream &print(raw_ostream &O, - const TargetRegisterInfo *TRI = nullptr) const; + void dump(const TargetRegisterInfo *TRI = nullptr) const; }; template <> @@ -459,12 +458,7 @@ class TargetRegisterInfo; /// edge occurs first. void biasCriticalPath(); - void dump(const ScheduleDAG *G) const; - void dumpAll(const ScheduleDAG *G) const; - raw_ostream &print(raw_ostream &O, - const SUnit *Entry = nullptr, - const SUnit *Exit = nullptr) const; - raw_ostream &print(raw_ostream &O, const ScheduleDAG *G) const; + void dumpAttributes() const; private: void ComputeDepth(); @@ -564,7 +558,7 @@ class TargetRegisterInfo; class ScheduleDAG { public: - const TargetMachine &TM; ///< Target processor + const LLVMTargetMachine &TM; ///< Target processor const TargetInstrInfo *TII; ///< Target instruction information const TargetRegisterInfo *TRI; ///< Target processor register info MachineFunction &MF; ///< Machine function @@ -597,7 +591,9 @@ class TargetRegisterInfo; virtual void viewGraph(const Twine &Name, const Twine &Title); virtual void viewGraph(); - virtual void dumpNode(const SUnit *SU) const = 0; + virtual void dumpNode(const SUnit &SU) const = 0; + virtual void dump() const = 0; + void dumpNodeName(const SUnit &SU) const; /// Returns a label for an SUnit node in a visualization of the ScheduleDAG. virtual std::string getGraphNodeLabel(const SUnit *SU) const = 0; @@ -614,6 +610,9 @@ class TargetRegisterInfo; unsigned VerifyScheduledDAG(bool isBottomUp); #endif + protected: + void dumpNodeAll(const SUnit &SU) const; + private: /// Returns the MCInstrDesc of this SDNode or NULL. const MCInstrDesc *getNodeDesc(const SDNode *Node) const; diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index 520a23846f6e..daad18125db9 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -327,7 +327,8 @@ namespace llvm { /// whole MachineFunction. By default does nothing. virtual void finalizeSchedule() {} - void dumpNode(const SUnit *SU) const override; + void dumpNode(const SUnit &SU) const override; + void dump() const override; /// Returns a label for a DAG node that points to an instruction. std::string getGraphNodeLabel(const SUnit *SU) const override; diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h index badf927d0e95..fbe559f25556 100644 --- a/include/llvm/CodeGen/SchedulerRegistry.h +++ b/include/llvm/CodeGen/SchedulerRegistry.h @@ -29,16 +29,19 @@ namespace llvm { class ScheduleDAGSDNodes; class SelectionDAGISel; -class RegisterScheduler : public MachinePassRegistryNode { +class RegisterScheduler + : public MachinePassRegistryNode< + ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level)> { public: using FunctionPassCtor = ScheduleDAGSDNodes *(*)(SelectionDAGISel*, CodeGenOpt::Level); - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; RegisterScheduler(const char *N, const char *D, FunctionPassCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) - { Registry.Add(this); } + : MachinePassRegistryNode(N, D, C) { + Registry.Add(this); + } ~RegisterScheduler() { Registry.Remove(this); } @@ -51,7 +54,7 @@ public: return (RegisterScheduler *)Registry.getList(); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 888f9425ff90..67fe87fc96af 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -28,7 +28,7 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" @@ -188,8 +188,8 @@ public: return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); } - ArrayRef getSDDbgValues(const SDNode *Node) { - DbgValMapType::iterator I = DbgValMap.find(Node); + ArrayRef getSDDbgValues(const SDNode *Node) const { + auto I = DbgValMap.find(Node); if (I != DbgValMap.end()) return I->second; return ArrayRef(); @@ -229,7 +229,7 @@ class SelectionDAG { LLVMContext *Context; CodeGenOpt::Level OptLevel; - DivergenceAnalysis * DA = nullptr; + LegacyDivergenceAnalysis * DA = nullptr; FunctionLoweringInfo * FLI = nullptr; /// The function-level optimization remark emitter. Used to emit remarks @@ -308,6 +308,9 @@ public: : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } + + private: + virtual void anchor(); }; /// When true, additional steps are taken to @@ -382,7 +385,7 @@ public: /// Prepare this SelectionDAG to process code in the given MachineFunction. void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - DivergenceAnalysis * Divergence); + LegacyDivergenceAnalysis * Divergence); void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { FLI = FuncInfo; @@ -471,7 +474,9 @@ public: return Root; } +#ifndef NDEBUG void VerifyDAGDiverence(); +#endif /// This iterates over the nodes in the SelectionDAG, folding /// certain types of nodes together, or eliminating superfluous nodes. The @@ -784,24 +789,6 @@ public: /// value assuming it was the smaller SrcTy value. SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); - /// Return an operation which will any-extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by any-extending the low four - /// lanes of the operand from i8 to i32. - SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - - /// Return an operation which will sign extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by sign extending the low four - /// lanes of the operand from i8 to i32. - SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - - /// Return an operation which will zero extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by zero extending the low four - /// lanes of the operand from i8 to i32. - SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - /// Convert Op, which must be of integer type, to the integer type VT, /// by using an extension appropriate for the target's /// BooleanContent for type OpVT or truncating it. @@ -945,41 +932,45 @@ public: Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo); - /// Helper function to make it easier to build SetCC's if you just - /// have an ISD::CondCode instead of an SDValue. - /// + /// Helper function to make it easier to build SetCC's if you just have an + /// ISD::CondCode instead of an SDValue. SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond) { assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && - "Cannot compare scalars to vectors"); + "Cannot compare scalars to vectors"); assert(LHS.getValueType().isVector() == VT.isVector() && - "Cannot compare scalars to vectors"); + "Cannot compare scalars to vectors"); assert(Cond != ISD::SETCC_INVALID && - "Cannot create a setCC of an invalid node."); + "Cannot create a setCC of an invalid node."); return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); } - /// Helper function to make it easier to build Select's if you just - /// have operands and don't want to check for vector. + /// Helper function to make it easier to build Select's if you just have + /// operands and don't want to check for vector. SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS) { assert(LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"); assert(VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"); - return getNode(Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, - Cond, LHS, RHS); + auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; + return getNode(Opcode, DL, VT, Cond, LHS, RHS); } - /// Helper function to make it easier to build SelectCC's if you - /// just have an ISD::CondCode instead of an SDValue. - /// + /// Helper function to make it easier to build SelectCC's if you just have an + /// ISD::CondCode instead of an SDValue. SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond) { - return getNode(ISD::SELECT_CC, DL, True.getValueType(), - LHS, RHS, True, False, getCondCode(Cond)); + return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, + False, getCondCode(Cond)); } + /// Try to simplify a select/vselect into 1 of its operands or a constant. + SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); + + /// Try to simplify a shift into 1 of its operands or a constant. + SDValue simplifyShift(SDValue X, SDValue Y); + /// VAArg produces a result and token chain, and takes a pointer /// and a source value as input. SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, @@ -1140,6 +1131,13 @@ public: /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. SDValue expandVACopy(SDNode *Node); + /// Returs an GlobalAddress of the function from the current module with + /// name matching the given ExternalSymbol. Additionally can provide the + /// matched function. + /// Panics the function doesn't exists. + SDValue getSymbolFunctionGlobalAddress(SDValue Op, + Function **TargetFunction = nullptr); + /// *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that @@ -1156,6 +1154,11 @@ public: SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef Ops); + /// *Mutate* the specified machine node's memory references to the provided + /// list. + void setNodeMemRefs(MachineSDNode *N, + ArrayRef NewMemRefs); + // Propagates the change in divergence to users void updateDivergence(SDNode * N); @@ -1346,7 +1349,7 @@ public: void AddDbgLabel(SDDbgLabel *DB); /// Get the debug values which reference the given SDNode. - ArrayRef GetDbgValues(const SDNode* SD) { + ArrayRef GetDbgValues(const SDNode* SD) const { return DbgInfo->getSDDbgValues(SD); } @@ -1429,15 +1432,15 @@ public: /// every vector element. /// Targets can implement the computeKnownBitsForTargetNode method in the /// TargetLowering class to allow target nodes to be understood. - void computeKnownBits(SDValue Op, KnownBits &Known, unsigned Depth = 0) const; + KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; /// Determine which bits of Op are known to be either zero or one and return /// them in Known. The DemandedElts argument allows us to only collect the /// known bits that are shared by the requested vector elements. /// Targets can implement the computeKnownBitsForTargetNode method in the /// TargetLowering class to allow target nodes to be understood. - void computeKnownBits(SDValue Op, KnownBits &Known, const APInt &DemandedElts, - unsigned Depth = 0) const; + KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, + unsigned Depth = 0) const; /// Used to represent the possible overflow behavior of an operation. /// Never: the operation cannot overflow. @@ -1484,8 +1487,15 @@ public: /// X|Cst == X+Cst iff X&Cst = 0. bool isBaseWithConstantOffset(SDValue Op) const; - /// Test whether the given SDValue is known to never be NaN. - bool isKnownNeverNaN(SDValue Op) const; + /// Test whether the given SDValue is known to never be NaN. If \p SNaN is + /// true, returns if \p Op is known to never be a signaling NaN (it may still + /// be a qNaN). + bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; + + /// \returns true if \p Op is known to never be a signaling NaN. + bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { + return isKnownNeverNaN(Op, true, Depth); + } /// Test whether the given floating point SDValue is known to never be /// positive or negative zero. @@ -1503,6 +1513,27 @@ public: /// allow an 'add' to be transformed into an 'or'. bool haveNoCommonBitsSet(SDValue A, SDValue B) const; + /// Test whether \p V has a splatted value for all the demanded elements. + /// + /// On success \p UndefElts will indicate the elements that have UNDEF + /// values instead of the splat value, this is only guaranteed to be correct + /// for \p DemandedElts. + /// + /// NOTE: The function will return true for a demanded splat of UNDEF values. + bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts); + + /// Test whether \p V has a splatted value. + bool isSplatValue(SDValue V, bool AllowUndefs = false); + + /// Match a binop + shuffle pyramid that represents a horizontal reduction + /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p + /// Extract. The reduction must use one of the opcodes listed in /p + /// CandidateBinOps and on success /p BinOp will contain the matching opcode. + /// Returns the vector that is being reduced on, or SDValue() if a reduction + /// was not matched. + SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, + ArrayRef CandidateBinOps); + /// Utility function used by legalize and lowering to /// "unroll" a vector operation by splitting out the scalars and operating /// on each element individually. If the ResNE is 0, fully unroll the vector diff --git a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 580606441a9d..2b2c48d57bc0 100644 --- a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -45,18 +45,21 @@ public: IsIndexSignExt(IsIndexSignExt) {} SDValue getBase() { return Base; } + SDValue getBase() const { return Base; } SDValue getIndex() { return Index; } + SDValue getIndex() const { return Index; } - bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG) { + bool equalBaseIndex(const BaseIndexOffset &Other, + const SelectionDAG &DAG) const { int64_t Off; return equalBaseIndex(Other, DAG, Off); } - bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG, - int64_t &Off); + bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, + int64_t &Off) const; /// Parses tree in Ptr for base, index, offset addresses. - static BaseIndexOffset match(LSBaseSDNode *N, const SelectionDAG &DAG); + static BaseIndexOffset match(const LSBaseSDNode *N, const SelectionDAG &DAG); }; } // end namespace llvm diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index 86df0af7303f..6758c55c696a 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -132,6 +132,7 @@ public: OPC_CheckChild2Same, OPC_CheckChild3Same, OPC_CheckPatternPredicate, OPC_CheckPredicate, + OPC_CheckPredicateWithOperands, OPC_CheckOpcode, OPC_SwitchOpcode, OPC_CheckType, @@ -267,6 +268,17 @@ public: llvm_unreachable("Tblgen should generate the implementation of this!"); } + /// CheckNodePredicateWithOperands - This function is generated by tblgen in + /// the target. + /// It runs node predicate number PredNo and returns true if it succeeds or + /// false if it fails. The number is a private implementation detail to the + /// code tblgen produces. + virtual bool CheckNodePredicateWithOperands( + SDNode *N, unsigned PredNo, + const SmallVectorImpl &Operands) const { + llvm_unreachable("Tblgen should generate the implementation of this!"); + } + virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl > &Result) { diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 1af22185d366..10f284179084 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -672,6 +672,12 @@ public: case ISD::STRICT_FLOG2: case ISD::STRICT_FRINT: case ISD::STRICT_FNEARBYINT: + case ISD::STRICT_FMAXNUM: + case ISD::STRICT_FMINNUM: + case ISD::STRICT_FCEIL: + case ISD::STRICT_FFLOOR: + case ISD::STRICT_FROUND: + case ISD::STRICT_FTRUNC: return true; } } @@ -1589,15 +1595,38 @@ bool isAllOnesConstant(SDValue V); /// Returns true if \p V is a constant integer one. bool isOneConstant(SDValue V); +/// Return the non-bitcasted source operand of \p V if it exists. +/// If \p V is not a bitcasted value, it is returned as-is. +SDValue peekThroughBitcasts(SDValue V); + +/// Return the non-bitcasted and one-use source operand of \p V if it exists. +/// If \p V is not a bitcasted one-use value, it is returned as-is. +SDValue peekThroughOneUseBitcasts(SDValue V); + /// Returns true if \p V is a bitwise not operation. Assumes that an all ones /// constant is canonicalized to be operand 1. bool isBitwiseNot(SDValue V); /// Returns the SDNode if it is a constant splat BuildVector or constant int. -ConstantSDNode *isConstOrConstSplat(SDValue N); +ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false); /// Returns the SDNode if it is a constant splat BuildVector or constant float. -ConstantFPSDNode *isConstOrConstSplatFP(SDValue N); +ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); + +/// Return true if the value is a constant 0 integer or a splatted vector of +/// a constant 0 integer (with no undefs). +/// Build vector implicit truncation is not an issue for null values. +bool isNullOrNullSplat(SDValue V); + +/// Return true if the value is a constant 1 integer or a splatted vector of a +/// constant 1 integer (with no undefs). +/// Does not permit build vector implicit truncation. +bool isOneOrOneSplat(SDValue V); + +/// Return true if the value is a constant -1 integer or a splatted vector of a +/// constant -1 integer (with no undefs). +/// Does not permit build vector implicit truncation. +bool isAllOnesOrAllOnesSplat(SDValue V); class GlobalAddressSDNode : public SDNode { friend class SelectionDAG; @@ -2113,12 +2142,15 @@ public: MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} - // In the both nodes address is Op1, mask is Op2: - // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value - // MaskedStoreSDNode (Chain, ptr, mask, data) + // MaskedLoadSDNode (Chain, ptr, mask, passthru) + // MaskedStoreSDNode (Chain, data, ptr, mask) // Mask is a vector of i1 elements - const SDValue &getBasePtr() const { return getOperand(1); } - const SDValue &getMask() const { return getOperand(2); } + const SDValue &getBasePtr() const { + return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2); + } + const SDValue &getMask() const { + return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); + } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD || @@ -2143,7 +2175,10 @@ public: return static_cast(LoadSDNodeBits.ExtTy); } - const SDValue &getSrc0() const { return getOperand(3); } + const SDValue &getBasePtr() const { return getOperand(1); } + const SDValue &getMask() const { return getOperand(2); } + const SDValue &getPassThru() const { return getOperand(3); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; } @@ -2175,7 +2210,9 @@ public: /// memory at base_addr. bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } - const SDValue &getValue() const { return getOperand(3); } + const SDValue &getValue() const { return getOperand(1); } + const SDValue &getBasePtr() const { return getOperand(2); } + const SDValue &getMask() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSTORE; @@ -2201,7 +2238,6 @@ public: const SDValue &getBasePtr() const { return getOperand(3); } const SDValue &getIndex() const { return getOperand(4); } const SDValue &getMask() const { return getOperand(2); } - const SDValue &getValue() const { return getOperand(1); } const SDValue &getScale() const { return getOperand(5); } static bool classof(const SDNode *N) { @@ -2220,6 +2256,8 @@ public: EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} + const SDValue &getPassThru() const { return getOperand(1); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MGATHER; } @@ -2235,6 +2273,8 @@ public: EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} + const SDValue &getValue() const { return getOperand(1); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSCATTER; } @@ -2243,32 +2283,60 @@ public: /// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. +/// +/// Note that the only supported way to set the `memoperands` is by calling the +/// `SelectionDAG::setNodeMemRefs` function as the memory management happens +/// inside the DAG rather than in the node. class MachineSDNode : public SDNode { -public: - using mmo_iterator = MachineMemOperand **; - private: friend class SelectionDAG; MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs) {} - /// Memory reference descriptions for this instruction. - mmo_iterator MemRefs = nullptr; - mmo_iterator MemRefsEnd = nullptr; + // We use a pointer union between a single `MachineMemOperand` pointer and + // a pointer to an array of `MachineMemOperand` pointers. This is null when + // the number of these is zero, the single pointer variant used when the + // number is one, and the array is used for larger numbers. + // + // The array is allocated via the `SelectionDAG`'s allocator and so will + // always live until the DAG is cleaned up and doesn't require ownership here. + // + // We can't use something simpler like `TinyPtrVector` here because `SDNode` + // subclasses aren't managed in a conforming C++ manner. See the comments on + // `SelectionDAG::MorphNodeTo` which details what all goes on, but the + // constraint here is that these don't manage memory with their constructor or + // destructor and can be initialized to a good state even if they start off + // uninitialized. + PointerUnion MemRefs = {}; + + // Note that this could be folded into the above `MemRefs` member if doing so + // is advantageous at some point. We don't need to store this in most cases. + // However, at the moment this doesn't appear to make the allocation any + // smaller and makes the code somewhat simpler to read. + int NumMemRefs = 0; public: - mmo_iterator memoperands_begin() const { return MemRefs; } - mmo_iterator memoperands_end() const { return MemRefsEnd; } - bool memoperands_empty() const { return MemRefsEnd == MemRefs; } + using mmo_iterator = ArrayRef::const_iterator; + + ArrayRef memoperands() const { + // Special case the common cases. + if (NumMemRefs == 0) + return {}; + if (NumMemRefs == 1) + return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); + + // Otherwise we have an actual array. + return makeArrayRef(MemRefs.get(), NumMemRefs); + } + mmo_iterator memoperands_begin() const { return memoperands().begin(); } + mmo_iterator memoperands_end() const { return memoperands().end(); } + bool memoperands_empty() const { return memoperands().empty(); } - /// Assign this MachineSDNodes's memory reference descriptor - /// list. This does not transfer ownership. - void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { - for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI) - assert(*MMI && "Null mem ref detected!"); - MemRefs = NewMemRefs; - MemRefsEnd = NewMemRefsEnd; + /// Clear out the memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; } static bool classof(const SDNode *N) { @@ -2405,17 +2473,32 @@ namespace ISD { cast(N)->getAddressingMode() == ISD::UNINDEXED; } + /// Return true if the node is a math/logic binary operator. This corresponds + /// to the IR function of the same name. + inline bool isBinaryOp(const SDNode *N) { + auto Op = N->getOpcode(); + return (Op == ISD::ADD || Op == ISD::SUB || Op == ISD::MUL || + Op == ISD::AND || Op == ISD::OR || Op == ISD::XOR || + Op == ISD::SHL || Op == ISD::SRL || Op == ISD::SRA || + Op == ISD::SDIV || Op == ISD::UDIV || Op == ISD::SREM || + Op == ISD::UREM || Op == ISD::FADD || Op == ISD::FSUB || + Op == ISD::FMUL || Op == ISD::FDIV || Op == ISD::FREM); + } + /// Attempt to match a unary predicate against a scalar/splat constant or /// every element of a constant BUILD_VECTOR. + /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. bool matchUnaryPredicate(SDValue Op, - std::function Match); + std::function Match, + bool AllowUndefs = false); /// Attempt to match a binary predicate against a pair of scalar/splat /// constants or every element of a pair of constant BUILD_VECTORs. + /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. bool matchBinaryPredicate( SDValue LHS, SDValue RHS, - std::function Match); - + std::function Match, + bool AllowUndefs = false); } // end namespace ISD } // end namespace llvm diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 334267d9828b..8c8a7be459fd 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -413,8 +413,14 @@ class raw_ostream; /// Returns the base index for the given instruction. SlotIndex getInstructionIndex(const MachineInstr &MI) const { // Instructions inside a bundle have the same number as the bundle itself. - const MachineInstr &BundleStart = *getBundleStart(MI.getIterator()); - Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleStart); + auto BundleStart = getBundleStart(MI.getIterator()); + auto BundleEnd = getBundleEnd(MI.getIterator()); + // Use the first non-debug instruction in the bundle to get SlotIndex. + const MachineInstr &BundleNonDebug = + *skipDebugInstructionsForward(BundleStart, BundleEnd); + assert(!BundleNonDebug.isDebugInstr() && + "Could not use a debug instruction to query mi2iMap."); + Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleNonDebug); assert(itr != mi2iMap.end() && "Instruction not found in maps."); return itr->second; } @@ -442,7 +448,7 @@ class raw_ostream; /// MI is not required to have an index. SlotIndex getIndexBefore(const MachineInstr &MI) const { const MachineBasicBlock *MBB = MI.getParent(); - assert(MBB && "MI must be inserted inna basic block"); + assert(MBB && "MI must be inserted in a basic block"); MachineBasicBlock::const_iterator I = MI, B = MBB->begin(); while (true) { if (I == B) @@ -459,7 +465,7 @@ class raw_ostream; /// MI is not required to have an index. SlotIndex getIndexAfter(const MachineInstr &MI) const { const MachineBasicBlock *MBB = MI.getParent(); - assert(MBB && "MI must be inserted inna basic block"); + assert(MBB && "MI must be inserted in a basic block"); MachineBasicBlock::const_iterator I = MI, E = MBB->end(); while (true) { ++I; @@ -674,7 +680,7 @@ class raw_ostream; idx2MBBMap.push_back(IdxMBBPair(startIdx, mbb)); renumberIndexes(newItr); - llvm::sort(idx2MBBMap.begin(), idx2MBBMap.end(), Idx2MBBCompare()); + llvm::sort(idx2MBBMap, Idx2MBBCompare()); } /// Free the resources that were required to maintain a SlotIndex. diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index e584a4136e4f..8be9ae378557 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -236,25 +236,6 @@ public: FnInfos.clear(); } - /// Generate a stackmap record for a stackmap instruction. - /// - /// MI must be a raw STACKMAP, not a PATCHPOINT. - void recordStackMap(const MachineInstr &MI); - - /// Generate a stackmap record for a patchpoint instruction. - void recordPatchPoint(const MachineInstr &MI); - - /// Generate a stackmap record for a statepoint instruction. - void recordStatepoint(const MachineInstr &MI); - - /// If there is any stack map data, create a stack map section and serialize - /// the map info into it. This clears the stack map data structures - /// afterwards. - void serializeToStackMapSection(); - -private: - static const char *WSMP; - using LocationVec = SmallVector; using LiveOutVec = SmallVector; using ConstantPool = MapVector; @@ -283,6 +264,31 @@ private: using FnInfoMap = MapVector; using CallsiteInfoList = std::vector; + /// Generate a stackmap record for a stackmap instruction. + /// + /// MI must be a raw STACKMAP, not a PATCHPOINT. + void recordStackMap(const MachineInstr &MI); + + /// Generate a stackmap record for a patchpoint instruction. + void recordPatchPoint(const MachineInstr &MI); + + /// Generate a stackmap record for a statepoint instruction. + void recordStatepoint(const MachineInstr &MI); + + /// If there is any stack map data, create a stack map section and serialize + /// the map info into it. This clears the stack map data structures + /// afterwards. + void serializeToStackMapSection(); + + /// Get call site info. + CallsiteInfoList &getCSInfos() { return CSInfos; } + + /// Get function info. + FnInfoMap &getFnInfos() { return FnInfos; } + +private: + static const char *WSMP; + AsmPrinter &AP; CallsiteInfoList CSInfos; ConstantPool ConstPool; diff --git a/include/llvm/CodeGen/TargetFrameLowering.h b/include/llvm/CodeGen/TargetFrameLowering.h index f8effee998e3..b4d1da941433 100644 --- a/include/llvm/CodeGen/TargetFrameLowering.h +++ b/include/llvm/CodeGen/TargetFrameLowering.h @@ -207,8 +207,11 @@ public: return false; } - /// Return true if the target needs to disable frame pointer elimination. - virtual bool noFramePointerElim(const MachineFunction &MF) const; + /// Return true if the target wants to keep the frame pointer regardless of + /// the function attribute "frame-pointer". + virtual bool keepFramePointer(const MachineFunction &MF) const { + return false; + } /// hasFP - Return true if the specified function should have a dedicated /// frame pointer register. For most targets this is true only if the function diff --git a/include/llvm/CodeGen/TargetInstrInfo.h b/include/llvm/CodeGen/TargetInstrInfo.h index b5bc561d834c..961b90e9bc12 100644 --- a/include/llvm/CodeGen/TargetInstrInfo.h +++ b/include/llvm/CodeGen/TargetInstrInfo.h @@ -246,14 +246,14 @@ public: } /// If the specified machine instruction has a load from a stack slot, - /// return true along with the FrameIndex of the loaded stack slot and the - /// machine mem operand containing the reference. + /// return true along with the FrameIndices of the loaded stack slot and the + /// machine mem operands containing the reference. /// If not, return false. Unlike isLoadFromStackSlot, this returns true for /// any instructions that loads from the stack. This is just a hint, as some /// cases may be missed. - virtual bool hasLoadFromStackSlot(const MachineInstr &MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const; + virtual bool hasLoadFromStackSlot( + const MachineInstr &MI, + SmallVectorImpl &Accesses) const; /// If the specified machine instruction is a direct /// store to a stack slot, return the virtual or physical register number of @@ -284,14 +284,14 @@ public: } /// If the specified machine instruction has a store to a stack slot, - /// return true along with the FrameIndex of the loaded stack slot and the - /// machine mem operand containing the reference. + /// return true along with the FrameIndices of the loaded stack slot and the + /// machine mem operands containing the reference. /// If not, return false. Unlike isStoreToStackSlot, /// this returns true for any instructions that stores to the /// stack. This is just a hint, as some cases may be missed. - virtual bool hasStoreToStackSlot(const MachineInstr &MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const; + virtual bool hasStoreToStackSlot( + const MachineInstr &MI, + SmallVectorImpl &Accesses) const; /// Return true if the specified machine instruction /// is a copy of one stack slot to another and has no other effect. @@ -846,15 +846,33 @@ public: llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!"); } +protected: + /// Target-dependent implemenation for IsCopyInstr. /// If the specific machine instruction is a instruction that moves/copies /// value from one register to another register return true along with /// @Source machine operand and @Destination machine operand. - virtual bool isCopyInstr(const MachineInstr &MI, - const MachineOperand *&SourceOpNum, - const MachineOperand *&Destination) const { + virtual bool isCopyInstrImpl(const MachineInstr &MI, + const MachineOperand *&Source, + const MachineOperand *&Destination) const { return false; } +public: + /// If the specific machine instruction is a instruction that moves/copies + /// value from one register to another register return true along with + /// @Source machine operand and @Destination machine operand. + /// For COPY-instruction the method naturally returns true, for all other + /// instructions the method calls target-dependent implementation. + bool isCopyInstr(const MachineInstr &MI, const MachineOperand *&Source, + const MachineOperand *&Destination) const { + if (MI.isCopy()) { + Destination = &MI.getOperand(0); + Source = &MI.getOperand(1); + return true; + } + return isCopyInstrImpl(MI, Source, Destination); + } + /// Store the specified register of the given register class to the specified /// stack frame index. The store instruction is to be added to the given /// machine basic block before the specified machine instruction. If isKill @@ -1063,7 +1081,7 @@ public: /// getAddressSpaceForPseudoSourceKind - Given the kind of memory /// (e.g. stack) the target returns the corresponding address space. virtual unsigned - getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const { + getAddressSpaceForPseudoSourceKind(unsigned Kind) const { return 0; } @@ -1118,11 +1136,11 @@ public: return false; } - /// Get the base register and byte offset of an instruction that reads/writes + /// Get the base operand and byte offset of an instruction that reads/writes /// memory. - virtual bool getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, - int64_t &Offset, - const TargetRegisterInfo *TRI) const { + virtual bool getMemOperandWithOffset(MachineInstr &MI, + MachineOperand *&BaseOp, int64_t &Offset, + const TargetRegisterInfo *TRI) const { return false; } @@ -1146,8 +1164,8 @@ public: /// or /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); /// to TargetPassConfig::createMachineScheduler() to have an effect. - virtual bool shouldClusterMemOps(MachineInstr &FirstLdSt, unsigned BaseReg1, - MachineInstr &SecondLdSt, unsigned BaseReg2, + virtual bool shouldClusterMemOps(MachineOperand &BaseOp1, + MachineOperand &BaseOp2, unsigned NumLoads) const { llvm_unreachable("target did not implement shouldClusterMemOps()"); } @@ -1617,10 +1635,11 @@ public: "Target didn't implement TargetInstrInfo::getOutliningType!"); } - /// Returns target-defined flags defining properties of the MBB for - /// the outliner. - virtual unsigned getMachineOutlinerMBBFlags(MachineBasicBlock &MBB) const { - return 0x0; + /// Optional target hook that returns true if \p MBB is safe to outline from, + /// and returns any target-specific information in \p Flags. + virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, + unsigned &Flags) const { + return true; } /// Insert a custom frame for outlined functions. diff --git a/include/llvm/CodeGen/TargetLowering.h b/include/llvm/CodeGen/TargetLowering.h index 40540bd6e1ff..23dbaac03ebe 100644 --- a/include/llvm/CodeGen/TargetLowering.h +++ b/include/llvm/CodeGen/TargetLowering.h @@ -29,7 +29,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/RuntimeLibcalls.h" @@ -163,6 +163,7 @@ public: LLOnly, // Expand the (load) instruction into just a load-linked, which has // greater atomic guarantees than a normal load. CmpXChg, // Expand the instruction into cmpxchg; used by at least X86. + MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop. }; /// Enum that specifies when a multiplication should be expanded. @@ -268,6 +269,14 @@ public: return true; } + /// Return true if it is profitable to convert a select of FP constants into + /// a constant pool load whose address depends on the select condition. The + /// parameter may be used to differentiate a select with FP compare from + /// integer compare. + virtual bool reduceSelectOfFPConstantLoads(bool IsFPSetCC) const { + return true; + } + /// Return true if multiple condition registers are available. bool hasMultipleConditionRegisters() const { return HasMultipleConditionRegisters; @@ -278,7 +287,7 @@ public: /// Return the preferred vector type legalization action. virtual TargetLoweringBase::LegalizeTypeAction - getPreferredVectorAction(EVT VT) const { + getPreferredVectorAction(MVT VT) const { // The default action for one element vectors is to scalarize if (VT.getVectorNumElements() == 1) return TypeScalarizeVector; @@ -545,6 +554,12 @@ public: return false; } + /// Return true if inserting a scalar into a variable element of an undef + /// vector is more efficiently handled by splatting the scalar instead. + virtual bool shouldSplatInsEltVarIndex(EVT) const { + return false; + } + /// Return true if target supports floating point exceptions. bool hasFloatingPointExceptions() const { return HasFloatingPointExceptions; @@ -790,6 +805,38 @@ public: return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op]; } + /// Custom method defined by each target to indicate if an operation which + /// may require a scale is supported natively by the target. + /// If not, the operation is illegal. + virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT, + unsigned Scale) const { + return false; + } + + /// Some fixed point operations may be natively supported by the target but + /// only for specific scales. This method allows for checking + /// if the width is supported by the target for a given operation that may + /// depend on scale. + LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, + unsigned Scale) const { + auto Action = getOperationAction(Op, VT); + if (Action != Legal) + return Action; + + // This operation is supported in this type but may only work on specific + // scales. + bool Supported; + switch (Op) { + default: + llvm_unreachable("Unexpected fixed point operation."); + case ISD::SMULFIX: + Supported = isSupportedFixedPointOperation(Op, VT, Scale); + break; + } + + return Supported ? Action : Expand; + } + LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const { unsigned EqOpc; switch (Op) { @@ -798,6 +845,7 @@ public: case ISD::STRICT_FSUB: EqOpc = ISD::FSUB; break; case ISD::STRICT_FMUL: EqOpc = ISD::FMUL; break; case ISD::STRICT_FDIV: EqOpc = ISD::FDIV; break; + case ISD::STRICT_FREM: EqOpc = ISD::FREM; break; case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break; case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break; case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break; @@ -811,6 +859,12 @@ public: case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break; case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break; case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break; + case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break; + case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break; + case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break; + case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; + case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; + case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; } auto Action = getOperationAction(EqOpc, VT); @@ -1199,13 +1253,15 @@ public: /// reduce runtime. virtual bool ShouldShrinkFPConstant(EVT) const { return true; } - // Return true if it is profitable to reduce the given load node to a smaller - // type. - // - // e.g. (i16 (trunc (i32 (load x))) -> i16 load x should be performed - virtual bool shouldReduceLoadWidth(SDNode *Load, - ISD::LoadExtType ExtTy, + /// Return true if it is profitable to reduce a load to a smaller type. + /// Example: (i16 (trunc (i32 (load x))) -> i16 load x + virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const { + // By default, assume that it is cheaper to extract a subvector from a wide + // vector load rather than creating multiple narrow vector loads. + if (NewVT.isVector() && !Load->hasOneUse()) + return false; + return true; } @@ -1428,6 +1484,12 @@ public: return PrefLoopAlignment; } + /// Should loops be aligned even when the function is marked OptSize (but not + /// MinSize). + virtual bool alignLoopsWithOptSize() const { + return false; + } + /// If the target has a standard location for the stack protector guard, /// returns the address of that location. Otherwise, returns nullptr. /// DEPRECATED: please override useLoadStackGuardNode and customize @@ -1549,6 +1611,26 @@ public: llvm_unreachable("Store conditional unimplemented on this target"); } + /// Perform a masked atomicrmw using a target-specific intrinsic. This + /// represents the core LL/SC loop which will be lowered at a late stage by + /// the backend. + virtual Value *emitMaskedAtomicRMWIntrinsic(IRBuilder<> &Builder, + AtomicRMWInst *AI, + Value *AlignedAddr, Value *Incr, + Value *Mask, Value *ShiftAmt, + AtomicOrdering Ord) const { + llvm_unreachable("Masked atomicrmw expansion unimplemented on this target"); + } + + /// Perform a masked cmpxchg using a target-specific intrinsic. This + /// represents the core LL/SC loop which will be lowered at a late stage by + /// the backend. + virtual Value *emitMaskedAtomicCmpXchgIntrinsic( + IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, + Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { + llvm_unreachable("Masked cmpxchg expansion unimplemented on this target"); + } + /// Inserts in the IR a target-specific intrinsic specifying a fence. /// It is called by AtomicExpandPass before expanding an /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad @@ -1625,11 +1707,11 @@ public: return AtomicExpansionKind::None; } - /// Returns true if the given atomic cmpxchg should be expanded by the - /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence - /// (through emitLoadLinked() and emitStoreConditional()). - virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { - return false; + /// Returns how the given atomic cmpxchg should be expanded by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + return AtomicExpansionKind::None; } /// Returns how the IR-level AtomicExpand pass should expand the given @@ -1687,6 +1769,25 @@ public: return false; } + /// Return true if it is profitable to transform an integer + /// multiplication-by-constant into simpler operations like shifts and adds. + /// This may be true if the target does not directly support the + /// multiplication operation for the specified type or the sequence of simpler + /// ops is faster than the multiply. + virtual bool decomposeMulByConstant(EVT VT, SDValue C) const { + return false; + } + + /// Return true if it is more correct/profitable to use strict FP_TO_INT + /// conversion operations - canonicalizing the FP source value instead of + /// converting all cases and then selecting based on value. + /// This may be true if the target throws exceptions for out of bounds + /// conversions or has fast FP CMOV. + virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, + bool IsSigned) const { + return false; + } + //===--------------------------------------------------------------------===// // TargetLowering Configuration Methods - These methods should be invoked by // the derived class constructor to configure this object for the target. @@ -2015,6 +2116,14 @@ public: return true; } + /// Return true if the specified immediate is legal for the value input of a + /// store instruction. + virtual bool isLegalStoreImmediate(int64_t Value) const { + // Default implementation assumes that at least 0 works since it is likely + // that a zero register exists or a zero immediate is allowed. + return Value == 0; + } + /// Return true if it's significantly cheaper to shift a vector by a uniform /// scalar than by an amount which will vary across each lane. On x86, for /// example, there is a "psllw" instruction for the former case, but no simple @@ -2046,10 +2155,12 @@ public: case ISD::UADDO: case ISD::ADDC: case ISD::ADDE: + case ISD::SADDSAT: + case ISD::UADDSAT: case ISD::FMINNUM: case ISD::FMAXNUM: - case ISD::FMINNAN: - case ISD::FMAXNAN: + case ISD::FMINIMUM: + case ISD::FMAXIMUM: return true; default: return false; } @@ -2153,6 +2264,12 @@ public: return false; } + /// Return true if sign-extension from FromTy to ToTy is cheaper than + /// zero-extension. + virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const { + return false; + } + /// Return true if the target supplies and combines to a paired load /// two loaded values of type LoadedType next to each other in memory. /// RequiredAlignment gives the minimal alignment constraints that must be met @@ -2292,6 +2409,12 @@ public: return false; } + /// Try to convert an extract element of a vector binary operation into an + /// extract element followed by a scalar operation. + virtual bool shouldScalarizeBinop(SDValue VecOp) const { + return false; + } + // Return true if it is profitable to use a scalar input to a BUILD_VECTOR // even if the vector itself has multiple uses. virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const { @@ -2648,7 +2771,7 @@ public: virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, - DivergenceAnalysis *DA) const { + LegacyDivergenceAnalysis *DA) const { return false; } @@ -2774,36 +2897,33 @@ public: bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded, TargetLoweringOpt &TLO) const; - /// Helper for SimplifyDemandedBits that can simplify an operation with - /// multiple uses. This function simplifies operand \p OpIdx of \p User and - /// then updates \p User with the simplified version. No other uses of - /// \p OpIdx are updated. If \p User is the only user of \p OpIdx, this - /// function behaves exactly like function SimplifyDemandedBits declared - /// below except that it also updates the DAG by calling - /// DCI.CommitTargetLoweringOpt. - bool SimplifyDemandedBits(SDNode *User, unsigned OpIdx, const APInt &Demanded, - DAGCombinerInfo &DCI, TargetLoweringOpt &TLO) const; - - /// Look at Op. At this point, we know that only the DemandedMask bits of the + /// Look at Op. At this point, we know that only the DemandedBits bits of the /// result of Op are ever used downstream. If we can use this information to /// simplify Op, create a new simplified DAG node and return true, returning /// the original and new nodes in Old and New. Otherwise, analyze the /// expression and return a mask of KnownOne and KnownZero bits for the /// expression (used to simplify the caller). The KnownZero/One bits may only - /// be accurate for those bits in the DemandedMask. + /// be accurate for those bits in the Demanded masks. /// \p AssumeSingleUse When this parameter is true, this function will /// attempt to simplify \p Op even if there are multiple uses. /// Callers are responsible for correctly updating the DAG based on the /// results of this function, because simply replacing replacing TLO.Old /// with TLO.New will be incorrect when this parameter is true and TLO.Old /// has multiple uses. - bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, - KnownBits &Known, - TargetLoweringOpt &TLO, + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts, KnownBits &Known, + TargetLoweringOpt &TLO, unsigned Depth = 0, + bool AssumeSingleUse = false) const; + + /// Helper wrapper around SimplifyDemandedBits, demanding all elements. + /// Adds Op back to the worklist upon success. + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth = 0, bool AssumeSingleUse = false) const; - /// Helper wrapper around SimplifyDemandedBits + /// Helper wrapper around SimplifyDemandedBits. + /// Adds Op back to the worklist upon success. bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, DAGCombinerInfo &DCI) const; @@ -2826,7 +2946,8 @@ public: TargetLoweringOpt &TLO, unsigned Depth = 0, bool AssumeSingleUse = false) const; - /// Helper wrapper around SimplifyDemandedVectorElts + /// Helper wrapper around SimplifyDemandedVectorElts. + /// Adds Op back to the worklist upon success. bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, DAGCombinerInfo &DCI) const; @@ -2863,11 +2984,30 @@ public: /// elements, returning true on success. Otherwise, analyze the expression and /// return a mask of KnownUndef and KnownZero elements for the expression /// (used to simplify the caller). The KnownUndef/Zero elements may only be - /// accurate for those bits in the DemandedMask + /// accurate for those bits in the DemandedMask. virtual bool SimplifyDemandedVectorEltsForTargetNode( SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0) const; + /// Attempt to simplify any target nodes based on the demanded bits/elts, + /// returning true on success. Otherwise, analyze the + /// expression and return a mask of KnownOne and KnownZero bits for the + /// expression (used to simplify the caller). The KnownZero/One bits may only + /// be accurate for those bits in the Demanded masks. + virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, + const APInt &DemandedBits, + const APInt &DemandedElts, + KnownBits &Known, + TargetLoweringOpt &TLO, + unsigned Depth = 0) const; + + /// If \p SNaN is false, \returns true if \p Op is known to never be any + /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling + /// NaN. + virtual bool isKnownNeverNaNForTargetNode(SDValue Op, + const SelectionDAG &DAG, + bool SNaN = false, + unsigned Depth = 0) const; struct DAGCombinerInfo { void *DC; // The DAG Combiner object. CombineLevel Level; @@ -2935,12 +3075,25 @@ public: /// virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; - /// Return true if it is profitable to move a following shift through this - // node, adjusting any immediate operands as necessary to preserve semantics. - // This transformation may not be desirable if it disrupts a particularly - // auspicious target-specific tree (e.g. bitfield extraction in AArch64). - // By default, it returns true. - virtual bool isDesirableToCommuteWithShift(const SDNode *N) const { + /// Return true if it is profitable to move this shift by a constant amount + /// though its operand, adjusting any immediate operands as necessary to + /// preserve semantics. This transformation may not be desirable if it + /// disrupts a particularly auspicious target-specific tree (e.g. bitfield + /// extraction in AArch64). By default, it returns true. + /// + /// @param N the shift node + /// @param Level the current DAGCombine legalization level. + virtual bool isDesirableToCommuteWithShift(const SDNode *N, + CombineLevel Level) const { + return true; + } + + /// Return true if it is profitable to fold a pair of shifts into a mask. + /// This is usually true on most targets. But some targets, like Thumb1, + /// have immediate shift instructions, but no immediate "and" instruction; + /// this makes the fold unprofitable. + virtual bool shouldFoldShiftPairToMask(const SDNode *N, + CombineLevel Level) const { return true; } @@ -3488,11 +3641,9 @@ public: //===--------------------------------------------------------------------===// // Div utility functions // - SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, - bool IsAfterLegalization, + SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl &Created) const; - SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, - bool IsAfterLegalization, + SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl &Created) const; /// Targets may override this function to provide custom SDIV lowering for @@ -3584,12 +3735,68 @@ public: SDValue LL = SDValue(), SDValue LH = SDValue(), SDValue RL = SDValue(), SDValue RH = SDValue()) const; + /// Expand funnel shift. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandFunnelShift(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand rotations. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandROT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Expand float(f32) to SINT(i64) conversion /// \param N Node to expand /// \param Result output after conversion /// \returns True, if the expansion was successful, false otherwise bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Expand float to UINT conversion + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand UINT(i64) to double(f64) conversion + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs. + SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const; + + /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTPOP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand CTLZ/CTLZ_ZERO_UNDEF nodes. Expands vector/scalar CTLZ nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTLZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand CTTZ/CTTZ_ZERO_UNDEF nodes. Expands vector/scalar CTTZ nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand ABS nodes. Expands vector/scalar ABS nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size)) + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Turn load of vector type into a load of the individual elements. /// \param LD load to expand /// \returns MERGE_VALUEs of the scalar loads with their chains. @@ -3627,6 +3834,15 @@ public: SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const; + /// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This + /// method accepts integers as its arguments. + SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const; + + /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts + /// integers as its arguments. + SDValue getExpandedFixedPointMultiplication(SDNode *Node, + SelectionDAG &DAG) const; + //===--------------------------------------------------------------------===// // Instruction Emitting Hooks // diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index f5c7fc824ab4..052d1f8bc686 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -90,6 +90,8 @@ public: const MCExpr *lowerRelativeReference(const GlobalValue *LHS, const GlobalValue *RHS, const TargetMachine &TM) const override; + + MCSection *getSectionForCommandLines() const override; }; class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { diff --git a/include/llvm/CodeGen/TargetPassConfig.h b/include/llvm/CodeGen/TargetPassConfig.h index 8f5c9cb8c3fa..3288711a335d 100644 --- a/include/llvm/CodeGen/TargetPassConfig.h +++ b/include/llvm/CodeGen/TargetPassConfig.h @@ -90,6 +90,19 @@ private: AnalysisID StartAfter = nullptr; AnalysisID StopBefore = nullptr; AnalysisID StopAfter = nullptr; + + unsigned StartBeforeInstanceNum = 0; + unsigned StartBeforeCount = 0; + + unsigned StartAfterInstanceNum = 0; + unsigned StartAfterCount = 0; + + unsigned StopBeforeInstanceNum = 0; + unsigned StopBeforeCount = 0; + + unsigned StopAfterInstanceNum = 0; + unsigned StopAfterCount = 0; + bool Started = true; bool Stopped = false; bool AddingMachinePasses = false; @@ -145,13 +158,13 @@ public: CodeGenOpt::Level getOptLevel() const; - /// Describe the status of the codegen - /// pipeline set by this target pass config. - /// Having a limited codegen pipeline means that options - /// have been used to restrict what codegen is doing. - /// In particular, that means that codegen won't emit - /// assembly code. - bool hasLimitedCodeGenPipeline() const; + /// Returns true if one of the `-start-after`, `-start-before`, `-stop-after` + /// or `-stop-before` options is set. + static bool hasLimitedCodeGenPipeline(); + + /// Returns true if none of the `-stop-before` and `-stop-after` options is + /// set. + static bool willCompleteCodeGenPipeline(); /// If hasLimitedCodeGenPipeline is true, this method /// returns a string with the name of the options, separated @@ -159,13 +172,6 @@ public: std::string getLimitedCodeGenPipelineReason(const char *Separator = "/") const; - /// Check if the codegen pipeline is limited in such a way that it - /// won't be complete. When the codegen pipeline is not complete, - /// this means it may not be possible to generate assembly from it. - bool willCompleteCodeGenPipeline() const { - return !hasLimitedCodeGenPipeline() || (!StopAfter && !StopBefore); - } - void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); } bool getEnableTailMerge() const { return EnableTailMerge; } diff --git a/include/llvm/CodeGen/TargetRegisterInfo.h b/include/llvm/CodeGen/TargetRegisterInfo.h index 55a8ba630a59..0fbff3137653 100644 --- a/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/include/llvm/CodeGen/TargetRegisterInfo.h @@ -510,6 +510,13 @@ public: /// markSuperRegs() and checkAllSuperRegsMarked() in this case. virtual BitVector getReservedRegs(const MachineFunction &MF) const = 0; + /// Returns false if we can't guarantee that Physreg, specified as an IR asm + /// clobber constraint, will be preserved across the statement. + virtual bool isAsmClobberable(const MachineFunction &MF, + unsigned PhysReg) const { + return true; + } + /// Returns true if PhysReg is unallocatable and constant throughout the /// function. Used by MachineRegisterInfo::isConstantPhysReg(). virtual bool isConstantPhysReg(unsigned PhysReg) const { return false; } @@ -817,13 +824,6 @@ public: // Do nothing. } - /// The creation of multiple copy hints have been implemented in - /// weightCalcHelper(), but since this affects so many tests for many - /// targets, this is temporarily disabled per default. THIS SHOULD BE - /// "GENERAL GOODNESS" and hopefully all targets will update their tests - /// and enable this soon. This hook should then be removed. - virtual bool enableMultipleCopyHints() const { return false; } - /// Allow the target to reverse allocation order of local live ranges. This /// will generally allocate shorter local live ranges first. For targets with /// many registers, this could reduce regalloc compile time by a large diff --git a/include/llvm/CodeGen/TargetSubtargetInfo.h b/include/llvm/CodeGen/TargetSubtargetInfo.h index 227e591f5a7d..968e4c4b8102 100644 --- a/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_TARGETSUBTARGETINFO_H #define LLVM_CODEGEN_TARGETSUBTARGETINFO_H +#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -144,6 +145,43 @@ public: return 0; } + /// Returns true if MI is a dependency breaking zero-idiom instruction for the + /// subtarget. + /// + /// This function also sets bits in Mask related to input operands that + /// are not in a data dependency relationship. There is one bit for each + /// machine operand; implicit operands follow explicit operands in the bit + /// representation used for Mask. An empty (i.e. a mask with all bits + /// cleared) means: data dependencies are "broken" for all the explicit input + /// machine operands of MI. + virtual bool isZeroIdiom(const MachineInstr *MI, APInt &Mask) const { + return false; + } + + /// Returns true if MI is a dependency breaking instruction for the subtarget. + /// + /// Similar in behavior to `isZeroIdiom`. However, it knows how to identify + /// all dependency breaking instructions (i.e. not just zero-idioms). + /// + /// As for `isZeroIdiom`, this method returns a mask of "broken" dependencies. + /// (See method `isZeroIdiom` for a detailed description of Mask). + virtual bool isDependencyBreaking(const MachineInstr *MI, APInt &Mask) const { + return isZeroIdiom(MI, Mask); + } + + /// Returns true if MI is a candidate for move elimination. + /// + /// A candidate for move elimination may be optimized out at register renaming + /// stage. Subtargets can specify the set of optimizable moves by + /// instantiating tablegen class `IsOptimizableRegisterMove` (see + /// llvm/Target/TargetInstrPredicate.td). + /// + /// SubtargetEmitter is responsible for processing all the definitions of class + /// IsOptimizableRegisterMove, and auto-generate an override for this method. + virtual bool isOptimizableRegisterMove(const MachineInstr *MI) const { + return false; + } + /// True if the subtarget should run MachineScheduler after aggressive /// coalescing. /// diff --git a/include/llvm/CodeGen/WasmEHFuncInfo.h b/include/llvm/CodeGen/WasmEHFuncInfo.h index 3ad6760d8813..219fff988f6e 100644 --- a/include/llvm/CodeGen/WasmEHFuncInfo.h +++ b/include/llvm/CodeGen/WasmEHFuncInfo.h @@ -14,13 +14,15 @@ #ifndef LLVM_CODEGEN_WASMEHFUNCINFO_H #define LLVM_CODEGEN_WASMEHFUNCINFO_H -#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/BasicBlock.h" namespace llvm { +enum EventTag { CPP_EXCEPTION = 0, C_LONGJMP = 1 }; + using BBOrMBB = PointerUnion; struct WasmEHFuncInfo { -- cgit v1.3