diff options
Diffstat (limited to 'include/llvm/CodeGen')
66 files changed, 2977 insertions, 2037 deletions
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index e3ce57ad18502..47201e2564e3c 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -16,8 +16,10 @@ #ifndef LLVM_CODEGEN_ASMPRINTER_H #define LLVM_CODEGEN_ASMPRINTER_H +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/Twine.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/DwarfStringPoolEntry.h" #include "llvm/IR/InlineAsm.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" @@ -29,6 +31,8 @@ class ByteStreamer; class GCStrategy; class Constant; class ConstantArray; +class DIE; +class DIEAbbrev; class GCMetadataPrinter; class GlobalValue; class GlobalVariable; @@ -46,11 +50,11 @@ class MCCFIInstruction; class MCContext; class MCExpr; class MCInst; -class MCInstrInfo; class MCSection; class MCStreamer; class MCSubtargetInfo; class MCSymbol; +class MCTargetOptions; class MDNode; class DwarfDebug; class Mangler; @@ -69,7 +73,6 @@ public: /// const MCAsmInfo *MAI; - const MCInstrInfo *MII; /// This is the context for the output file that we are streaming. This owns /// all of the global MC-related objects for the generated translation unit. MCContext &OutContext; @@ -77,7 +80,7 @@ public: /// This is the MCStreamer object for the file we are generating. This /// contains the transient state for the current translation unit that we are /// generating (such as the current section etc). - MCStreamer &OutStreamer; + std::unique_ptr<MCStreamer> OutStreamer; /// The current machine function. const MachineFunction *MF; @@ -99,7 +102,16 @@ public: /// default, this is equal to CurrentFnSym. MCSymbol *CurrentFnSymForSize; + /// Map global GOT equivalent MCSymbols to GlobalVariables and keep track of + /// its number of uses by other globals. + typedef std::pair<const GlobalVariable *, unsigned> GOTEquivUsePair; + MapVector<const MCSymbol *, GOTEquivUsePair> GlobalGOTEquivs; + private: + MCSymbol *CurrentFnBegin; + MCSymbol *CurrentFnEnd; + MCSymbol *CurExceptionSym; + // The garbage collection metadata printer table. void *GCMetadataPrinters; // Really a DenseMap. @@ -127,10 +139,10 @@ private: DwarfDebug *DD; protected: - explicit AsmPrinter(TargetMachine &TM, MCStreamer &Streamer); + explicit AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer); public: - virtual ~AsmPrinter(); + ~AsmPrinter() override; DwarfDebug *getDwarfDebug() { return DD; } DwarfDebug *getDwarfDebug() const { return DD; } @@ -143,6 +155,10 @@ public: /// unsigned getFunctionNumber() const; + MCSymbol *getFunctionBegin() const { return CurrentFnBegin; } + MCSymbol *getFunctionEnd() const { return CurrentFnEnd; } + MCSymbol *getCurExceptionSym(); + /// Return information about object file lowering. const TargetLoweringObjectFile &getObjFileLowering() const; @@ -184,7 +200,6 @@ public: /// Emit the specified function out to the OutStreamer. bool runOnMachineFunction(MachineFunction &MF) override { SetupMachineFunction(MF); - EmitFunctionHeader(); EmitFunctionBody(); return false; } @@ -197,9 +212,6 @@ public: /// runOnMachineFunction. void SetupMachineFunction(MachineFunction &MF); - /// This method emits the header for the current function. - void EmitFunctionHeader(); - /// This method emits the body and trailer for a function. void EmitFunctionBody(); @@ -238,16 +250,27 @@ public: /// void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const; - /// This method prints the label for the specified MachineBasicBlock, an - /// alignment (if present) and a comment describing it if appropriate. - void EmitBasicBlockStart(const MachineBasicBlock &MBB) const; - /// Lower the specified LLVM Constant to an MCExpr. const MCExpr *lowerConstant(const Constant *CV); /// \brief Print a general LLVM constant to the .s file. void EmitGlobalConstant(const Constant *CV); + /// \brief Unnamed constant global variables solely contaning a pointer to + /// another globals variable act like a global variable "proxy", or GOT + /// equivalents, i.e., it's only used to hold the address of the latter. One + /// optimization is to replace accesses to these proxies by using the GOT + /// entry for the final global instead. Hence, we select GOT equivalent + /// candidates among all the module global variables, avoid emitting them + /// unnecessarily and finally replace references to them by pc relative + /// accesses to GOT entries. + void computeGlobalGOTEquivs(Module &M); + + /// \brief Constant expressions using GOT equivalent globals may not be + /// eligible for PC relative GOT entry conversion, in such cases we need to + /// emit the proxies we previously omitted in EmitGlobalVariable. + void emitGlobalGOTEquivs(); + //===------------------------------------------------------------------===// // Overridable Hooks //===------------------------------------------------------------------===// @@ -271,6 +294,12 @@ public: /// function. virtual void EmitFunctionBodyEnd() {} + /// Targets can override this to emit stuff at the start of a basic block. + /// By default, this method prints the label for the specified + /// MachineBasicBlock, an alignment (if present) and a comment describing it + /// if appropriate. + virtual void EmitBasicBlockStart(const MachineBasicBlock &MBB) const; + /// Targets can override this to emit stuff at the end of a basic block. virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB) {} @@ -304,12 +333,7 @@ public: // Symbol Lowering Routines. //===------------------------------------------------------------------===// public: - /// Return the MCSymbol corresponding to the assembler temporary label with - /// the specified stem and unique ID. - MCSymbol *GetTempSymbol(Twine Name, unsigned ID) const; - - /// Return an assembler temporary label with the specified stem. - MCSymbol *GetTempSymbol(Twine Name) const; + MCSymbol *createTempSymbol(const Twine &Name) const; /// Return the MCSymbol for a private symbol with global value name as its /// base, with the specified suffix. @@ -397,42 +421,21 @@ public: /// Emit the 4-byte offset of Label from the start of its section. This can /// be done with a special directive if the target supports it (e.g. cygwin) /// or by emitting it as an offset from a label at the start of the section. + void emitSectionOffset(const MCSymbol *Label) const; + + /// Emit the 4-byte offset of a string from the start of its section. /// - /// SectionLabel is a temporary label emitted at the start of the section - /// that Label lives in. - void EmitSectionOffset(const MCSymbol *Label, - const MCSymbol *SectionLabel) const; + /// When possible, emit a DwarfStringPool section offset without any + /// relocations, and without using the symbol. Otherwise, defers to \a + /// emitSectionOffset(). + void emitDwarfStringOffset(DwarfStringPoolEntryRef S) const; /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified. virtual unsigned getISAEncoding() { return 0; } - /// Emit a dwarf register operation for describing - /// - a small value occupying only part of a register or - /// - a register representing only part of a value. - void EmitDwarfOpPiece(ByteStreamer &Streamer, unsigned SizeInBits, - unsigned OffsetInBits = 0) const; - - - /// \brief Emit a partial DWARF register operation. - /// \param MLoc the register - /// \param PieceSize size and - /// \param PieceOffset offset of the piece in bits, if this is one - /// piece of an aggregate value. - /// - /// If size and offset is zero an operation for the entire - /// register is emitted: Some targets do not provide a DWARF - /// register number for every register. If this is the case, this - /// function will attempt to emit a DWARF register by emitting a - /// piece of a super-register or by piecing together multiple - /// subregisters that alias the register. - void EmitDwarfRegOpPiece(ByteStreamer &BS, const MachineLocation &MLoc, - unsigned PieceSize = 0, - unsigned PieceOffset = 0) const; - /// EmitDwarfRegOp - Emit a dwarf register operation. - /// \param Indirect whether this is a register-indirect address - virtual void EmitDwarfRegOp(ByteStreamer &BS, const MachineLocation &MLoc, - bool Indirect) const; + virtual void EmitDwarfRegOp(ByteStreamer &BS, + const MachineLocation &MLoc) const; //===------------------------------------------------------------------===// // Dwarf Lowering Routines @@ -441,6 +444,12 @@ public: /// \brief Emit frame instruction to describe the layout of the frame. void emitCFIInstruction(const MCCFIInstruction &Inst) const; + /// \brief Emit Dwarf abbreviation table. + void emitDwarfAbbrevs(const std::vector<DIEAbbrev *>& Abbrevs) const; + + /// \brief Recursively emit Dwarf DIE tree. + void emitDwarfDIE(const DIE &Die) const; + //===------------------------------------------------------------------===// // Inline Asm Support //===------------------------------------------------------------------===// @@ -474,7 +483,7 @@ public: /// Let the target do anything it needs to do before emitting inlineasm. /// \p StartInfo - the subtarget info before parsing inline asm - virtual void emitInlineAsmStart(const MCSubtargetInfo &StartInfo) const; + virtual void emitInlineAsmStart() const; /// Let the target do anything it needs to do after emitting inlineasm. /// This callback can be used restore the original mode in case the @@ -491,11 +500,15 @@ private: mutable const MachineInstr *LastMI; mutable unsigned LastFn; mutable unsigned Counter; - mutable unsigned SetCounter; + + /// This method emits the header for the current function. + virtual void EmitFunctionHeader(); /// Emit a blob of inline asm to the output streamer. void - EmitInlineAsm(StringRef Str, const MDNode *LocMDNode = nullptr, + EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, + const MCTargetOptions &MCOptions, + const MDNode *LocMDNode = nullptr, InlineAsm::AsmDialect AsmDialect = InlineAsm::AD_ATT) const; /// This method formats and emits the specified machine instruction that is an diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h new file mode 100644 index 0000000000000..d07265560430e --- /dev/null +++ b/include/llvm/CodeGen/BasicTTIImpl.h @@ -0,0 +1,767 @@ +//===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file provides a helper that implements much of the TTI interface in +/// terms of the target-independent code generator and TargetLowering +/// interfaces. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_BASICTTIIMPL_H +#define LLVM_CODEGEN_BASICTTIIMPL_H + +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetTransformInfoImpl.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" + +namespace llvm { + +extern cl::opt<unsigned> PartialUnrollingThreshold; + +/// \brief Base class which can be used to help build a TTI implementation. +/// +/// This class provides as much implementation of the TTI interface as is +/// possible using the target independent parts of the code generator. +/// +/// In order to subclass it, your class must implement a getST() method to +/// return the subtarget, and a getTLI() method to return the target lowering. +/// We need these methods implemented in the derived class so that this class +/// doesn't have to duplicate storage for them. +template <typename T> +class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> { +private: + typedef TargetTransformInfoImplCRTPBase<T> BaseT; + typedef TargetTransformInfo TTI; + + /// Estimate the overhead of scalarizing an instruction. Insert and Extract + /// are set if the result needs to be inserted and/or extracted from vectors. + unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) { + assert(Ty->isVectorTy() && "Can only scalarize vectors"); + unsigned Cost = 0; + + for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) { + if (Insert) + Cost += static_cast<T *>(this) + ->getVectorInstrCost(Instruction::InsertElement, Ty, i); + if (Extract) + Cost += static_cast<T *>(this) + ->getVectorInstrCost(Instruction::ExtractElement, Ty, i); + } + + return Cost; + } + + /// Estimate the cost overhead of SK_Alternate shuffle. + unsigned getAltShuffleOverhead(Type *Ty) { + assert(Ty->isVectorTy() && "Can only shuffle vectors"); + unsigned Cost = 0; + // Shuffle cost is equal to the cost of extracting element from its argument + // plus the cost of inserting them onto the result vector. + + // e.g. <4 x float> has a mask of <0,5,2,7> i.e we need to extract from + // index 0 of first vector, index 1 of second vector,index 2 of first + // vector and finally index 3 of second vector and insert them at index + // <0,1,2,3> of result vector. + for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) { + Cost += static_cast<T *>(this) + ->getVectorInstrCost(Instruction::InsertElement, Ty, i); + Cost += static_cast<T *>(this) + ->getVectorInstrCost(Instruction::ExtractElement, Ty, i); + } + return Cost; + } + + /// \brief Local query method delegates up to T which *must* implement this! + const TargetSubtargetInfo *getST() const { + return static_cast<const T *>(this)->getST(); + } + + /// \brief Local query method delegates up to T which *must* implement this! + const TargetLoweringBase *getTLI() const { + return static_cast<const T *>(this)->getTLI(); + } + +protected: + explicit BasicTTIImplBase(const TargetMachine *TM) + : BaseT(TM->getDataLayout()) {} + +public: + // Provide value semantics. MSVC requires that we spell all of these out. + BasicTTIImplBase(const BasicTTIImplBase &Arg) + : BaseT(static_cast<const BaseT &>(Arg)) {} + BasicTTIImplBase(BasicTTIImplBase &&Arg) + : BaseT(std::move(static_cast<BaseT &>(Arg))) {} + BasicTTIImplBase &operator=(const BasicTTIImplBase &RHS) { + BaseT::operator=(static_cast<const BaseT &>(RHS)); + return *this; + } + BasicTTIImplBase &operator=(BasicTTIImplBase &&RHS) { + BaseT::operator=(std::move(static_cast<BaseT &>(RHS))); + return *this; + } + + /// \name Scalar TTI Implementations + /// @{ + + bool hasBranchDivergence() { return false; } + + bool isSourceOfDivergence(const Value *V) { return false; } + + bool isLegalAddImmediate(int64_t imm) { + return getTLI()->isLegalAddImmediate(imm); + } + + bool isLegalICmpImmediate(int64_t imm) { + return getTLI()->isLegalICmpImmediate(imm); + } + + bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, + bool HasBaseReg, int64_t Scale) { + TargetLoweringBase::AddrMode AM; + AM.BaseGV = BaseGV; + AM.BaseOffs = BaseOffset; + AM.HasBaseReg = HasBaseReg; + AM.Scale = Scale; + return getTLI()->isLegalAddressingMode(AM, Ty); + } + + int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, + bool HasBaseReg, int64_t Scale) { + TargetLoweringBase::AddrMode AM; + AM.BaseGV = BaseGV; + AM.BaseOffs = BaseOffset; + AM.HasBaseReg = HasBaseReg; + AM.Scale = Scale; + return getTLI()->getScalingFactorCost(AM, Ty); + } + + bool isTruncateFree(Type *Ty1, Type *Ty2) { + return getTLI()->isTruncateFree(Ty1, Ty2); + } + + bool isProfitableToHoist(Instruction *I) { + return getTLI()->isProfitableToHoist(I); + } + + bool isTypeLegal(Type *Ty) { + EVT VT = getTLI()->getValueType(Ty); + return getTLI()->isTypeLegal(VT); + } + + unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<const Value *> Arguments) { + return BaseT::getIntrinsicCost(IID, RetTy, Arguments); + } + + unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> ParamTys) { + if (IID == Intrinsic::cttz) { + if (getTLI()->isCheapToSpeculateCttz()) + return TargetTransformInfo::TCC_Basic; + return TargetTransformInfo::TCC_Expensive; + } + + if (IID == Intrinsic::ctlz) { + if (getTLI()->isCheapToSpeculateCtlz()) + return TargetTransformInfo::TCC_Basic; + return TargetTransformInfo::TCC_Expensive; + } + + return BaseT::getIntrinsicCost(IID, RetTy, ParamTys); + } + + unsigned getJumpBufAlignment() { return getTLI()->getJumpBufAlignment(); } + + unsigned getJumpBufSize() { return getTLI()->getJumpBufSize(); } + + bool shouldBuildLookupTables() { + const TargetLoweringBase *TLI = getTLI(); + return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || + TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other); + } + + bool haveFastSqrt(Type *Ty) { + const TargetLoweringBase *TLI = getTLI(); + EVT VT = TLI->getValueType(Ty); + return TLI->isTypeLegal(VT) && + TLI->isOperationLegalOrCustom(ISD::FSQRT, VT); + } + + unsigned getFPOpCost(Type *Ty) { + // By default, FP instructions are no more expensive since they are + // implemented in HW. Target specific TTI can override this. + return TargetTransformInfo::TCC_Basic; + } + + unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) { + const TargetLoweringBase *TLI = getTLI(); + switch (Opcode) { + default: break; + case Instruction::Trunc: { + if (TLI->isTruncateFree(OpTy, Ty)) + return TargetTransformInfo::TCC_Free; + return TargetTransformInfo::TCC_Basic; + } + case Instruction::ZExt: { + if (TLI->isZExtFree(OpTy, Ty)) + return TargetTransformInfo::TCC_Free; + return TargetTransformInfo::TCC_Basic; + } + } + + return BaseT::getOperationCost(Opcode, Ty, OpTy); + } + + void getUnrollingPreferences(Loop *L, TTI::UnrollingPreferences &UP) { + // This unrolling functionality is target independent, but to provide some + // motivation for its intended use, for x86: + + // According to the Intel 64 and IA-32 Architectures Optimization Reference + // Manual, Intel Core models and later have a loop stream detector (and + // associated uop queue) that can benefit from partial unrolling. + // The relevant requirements are: + // - The loop must have no more than 4 (8 for Nehalem and later) branches + // taken, and none of them may be calls. + // - The loop can have no more than 18 (28 for Nehalem and later) uops. + + // According to the Software Optimization Guide for AMD Family 15h + // Processors, models 30h-4fh (Steamroller and later) have a loop predictor + // and loop buffer which can benefit from partial unrolling. + // The relevant requirements are: + // - The loop must have fewer than 16 branches + // - The loop must have less than 40 uops in all executed loop branches + + // The number of taken branches in a loop is hard to estimate here, and + // benchmarking has revealed that it is better not to be conservative when + // estimating the branch count. As a result, we'll ignore the branch limits + // until someone finds a case where it matters in practice. + + unsigned MaxOps; + const TargetSubtargetInfo *ST = getST(); + if (PartialUnrollingThreshold.getNumOccurrences() > 0) + MaxOps = PartialUnrollingThreshold; + else if (ST->getSchedModel().LoopMicroOpBufferSize > 0) + MaxOps = ST->getSchedModel().LoopMicroOpBufferSize; + else + return; + + // Scan the loop: don't unroll loops with calls. + for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E; + ++I) { + BasicBlock *BB = *I; + + for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); J != JE; ++J) + if (isa<CallInst>(J) || isa<InvokeInst>(J)) { + ImmutableCallSite CS(J); + if (const Function *F = CS.getCalledFunction()) { + if (!static_cast<T *>(this)->isLoweredToCall(F)) + continue; + } + + return; + } + } + + // Enable runtime and partial unrolling up to the specified size. + UP.Partial = UP.Runtime = true; + UP.PartialThreshold = UP.PartialOptSizeThreshold = MaxOps; + } + + /// @} + + /// \name Vector TTI Implementations + /// @{ + + unsigned getNumberOfRegisters(bool Vector) { return 1; } + + unsigned getRegisterBitWidth(bool Vector) { return 32; } + + unsigned getMaxInterleaveFactor(unsigned VF) { return 1; } + + unsigned getArithmeticInstrCost( + unsigned Opcode, Type *Ty, + TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue, + TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue, + TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None, + TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None) { + // Check if any of the operands are vector operands. + const TargetLoweringBase *TLI = getTLI(); + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty); + + bool IsFloat = Ty->getScalarType()->isFloatingPointTy(); + // Assume that floating point arithmetic operations cost twice as much as + // integer operations. + unsigned OpCost = (IsFloat ? 2 : 1); + + if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. + // If the type is split to multiple registers, assume that there is some + // overhead to this. + // TODO: Once we have extract/insert subvector cost we need to use them. + if (LT.first > 1) + return LT.first * 2 * OpCost; + return LT.first * 1 * OpCost; + } + + if (!TLI->isOperationExpand(ISD, LT.second)) { + // If the operation is custom lowered then assume + // thare the code is twice as expensive. + return LT.first * 2 * OpCost; + } + + // Else, assume that we need to scalarize this op. + if (Ty->isVectorTy()) { + unsigned Num = Ty->getVectorNumElements(); + unsigned Cost = static_cast<T *>(this) + ->getArithmeticInstrCost(Opcode, Ty->getScalarType()); + // return the cost of multiple scalar invocation plus the cost of + // inserting + // and extracting the values. + return getScalarizationOverhead(Ty, true, true) + Num * Cost; + } + + // We don't know anything about this scalar instruction. + return OpCost; + } + + unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp) { + if (Kind == TTI::SK_Alternate) { + return getAltShuffleOverhead(Tp); + } + return 1; + } + + unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) { + const TargetLoweringBase *TLI = getTLI(); + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + std::pair<unsigned, MVT> SrcLT = TLI->getTypeLegalizationCost(Src); + std::pair<unsigned, MVT> DstLT = TLI->getTypeLegalizationCost(Dst); + + // Check for NOOP conversions. + if (SrcLT.first == DstLT.first && + SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) { + + // Bitcast between types that are legalized to the same type are free. + if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc) + return 0; + } + + if (Opcode == Instruction::Trunc && + TLI->isTruncateFree(SrcLT.second, DstLT.second)) + return 0; + + if (Opcode == Instruction::ZExt && + TLI->isZExtFree(SrcLT.second, DstLT.second)) + return 0; + + // If the cast is marked as legal (or promote) then assume low cost. + if (SrcLT.first == DstLT.first && + TLI->isOperationLegalOrPromote(ISD, DstLT.second)) + return 1; + + // Handle scalar conversions. + if (!Src->isVectorTy() && !Dst->isVectorTy()) { + + // Scalar bitcasts are usually free. + if (Opcode == Instruction::BitCast) + return 0; + + // Just check the op cost. If the operation is legal then assume it costs + // 1. + if (!TLI->isOperationExpand(ISD, DstLT.second)) + return 1; + + // Assume that illegal scalar instruction are expensive. + return 4; + } + + // Check vector-to-vector casts. + if (Dst->isVectorTy() && Src->isVectorTy()) { + + // If the cast is between same-sized registers, then the check is simple. + if (SrcLT.first == DstLT.first && + SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) { + + // Assume that Zext is done using AND. + if (Opcode == Instruction::ZExt) + return 1; + + // Assume that sext is done using SHL and SRA. + if (Opcode == Instruction::SExt) + return 2; + + // Just check the op cost. If the operation is legal then assume it + // costs + // 1 and multiply by the type-legalization overhead. + if (!TLI->isOperationExpand(ISD, DstLT.second)) + return SrcLT.first * 1; + } + + // If we are converting vectors and the operation is illegal, or + // if the vectors are legalized to different types, estimate the + // scalarization costs. + unsigned Num = Dst->getVectorNumElements(); + unsigned Cost = static_cast<T *>(this)->getCastInstrCost( + Opcode, Dst->getScalarType(), Src->getScalarType()); + + // Return the cost of multiple scalar invocation plus the cost of + // inserting and extracting the values. + return getScalarizationOverhead(Dst, true, true) + Num * Cost; + } + + // We already handled vector-to-vector and scalar-to-scalar conversions. + // This + // is where we handle bitcast between vectors and scalars. We need to assume + // that the conversion is scalarized in one way or another. + if (Opcode == Instruction::BitCast) + // Illegal bitcasts are done by storing and loading from a stack slot. + return (Src->isVectorTy() ? getScalarizationOverhead(Src, false, true) + : 0) + + (Dst->isVectorTy() ? getScalarizationOverhead(Dst, true, false) + : 0); + + llvm_unreachable("Unhandled cast"); + } + + unsigned getCFInstrCost(unsigned Opcode) { + // Branches are assumed to be predicted. + return 0; + } + + unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy) { + const TargetLoweringBase *TLI = getTLI(); + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + // Selects on vectors are actually vector selects. + if (ISD == ISD::SELECT) { + assert(CondTy && "CondTy must exist"); + if (CondTy->isVectorTy()) + ISD = ISD::VSELECT; + } + + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy); + + if (!(ValTy->isVectorTy() && !LT.second.isVector()) && + !TLI->isOperationExpand(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. Multiply + // by the type-legalization overhead. + return LT.first * 1; + } + + // Otherwise, assume that the cast is scalarized. + if (ValTy->isVectorTy()) { + unsigned Num = ValTy->getVectorNumElements(); + if (CondTy) + CondTy = CondTy->getScalarType(); + unsigned Cost = static_cast<T *>(this)->getCmpSelInstrCost( + Opcode, ValTy->getScalarType(), CondTy); + + // Return the cost of multiple scalar invocation plus the cost of + // inserting + // and extracting the values. + return getScalarizationOverhead(ValTy, true, false) + Num * Cost; + } + + // Unknown scalar opcode. + return 1; + } + + unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) { + std::pair<unsigned, MVT> LT = + getTLI()->getTypeLegalizationCost(Val->getScalarType()); + + return LT.first; + } + + unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) { + assert(!Src->isVoidTy() && "Invalid type"); + std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Src); + + // Assuming that all loads of legal types cost 1. + unsigned Cost = LT.first; + + if (Src->isVectorTy() && + Src->getPrimitiveSizeInBits() < LT.second.getSizeInBits()) { + // This is a vector load that legalizes to a larger type than the vector + // itself. Unless the corresponding extending load or truncating store is + // legal, then this will scalarize. + TargetLowering::LegalizeAction LA = TargetLowering::Expand; + EVT MemVT = getTLI()->getValueType(Src, true); + if (MemVT.isSimple() && MemVT != MVT::Other) { + if (Opcode == Instruction::Store) + LA = getTLI()->getTruncStoreAction(LT.second, MemVT.getSimpleVT()); + else + LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT); + } + + if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) { + // This is a vector load/store for some illegal type that is scalarized. + // We must account for the cost of building or decomposing the vector. + Cost += getScalarizationOverhead(Src, Opcode != Instruction::Store, + Opcode == Instruction::Store); + } + } + + return Cost; + } + + unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> Tys) { + unsigned ISD = 0; + switch (IID) { + default: { + // Assume that we need to scalarize this intrinsic. + unsigned ScalarizationCost = 0; + unsigned ScalarCalls = 1; + Type *ScalarRetTy = RetTy; + if (RetTy->isVectorTy()) { + ScalarizationCost = getScalarizationOverhead(RetTy, true, false); + ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); + ScalarRetTy = RetTy->getScalarType(); + } + SmallVector<Type *, 4> ScalarTys; + for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { + Type *Ty = Tys[i]; + if (Ty->isVectorTy()) { + ScalarizationCost += getScalarizationOverhead(Ty, false, true); + ScalarCalls = std::max(ScalarCalls, Ty->getVectorNumElements()); + Ty = Ty->getScalarType(); + } + ScalarTys.push_back(Ty); + } + if (ScalarCalls == 1) + return 1; // Return cost of a scalar intrinsic. Assume it to be cheap. + + unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost( + IID, ScalarRetTy, ScalarTys); + + return ScalarCalls * ScalarCost + ScalarizationCost; + } + // Look for intrinsics that can be lowered directly or turned into a scalar + // intrinsic call. + case Intrinsic::sqrt: + ISD = ISD::FSQRT; + break; + case Intrinsic::sin: + ISD = ISD::FSIN; + break; + case Intrinsic::cos: + ISD = ISD::FCOS; + break; + case Intrinsic::exp: + ISD = ISD::FEXP; + break; + case Intrinsic::exp2: + ISD = ISD::FEXP2; + break; + case Intrinsic::log: + ISD = ISD::FLOG; + break; + case Intrinsic::log10: + ISD = ISD::FLOG10; + break; + case Intrinsic::log2: + ISD = ISD::FLOG2; + break; + case Intrinsic::fabs: + ISD = ISD::FABS; + break; + case Intrinsic::minnum: + ISD = ISD::FMINNUM; + break; + case Intrinsic::maxnum: + ISD = ISD::FMAXNUM; + break; + case Intrinsic::copysign: + ISD = ISD::FCOPYSIGN; + break; + case Intrinsic::floor: + ISD = ISD::FFLOOR; + break; + case Intrinsic::ceil: + ISD = ISD::FCEIL; + break; + case Intrinsic::trunc: + ISD = ISD::FTRUNC; + break; + case Intrinsic::nearbyint: + ISD = ISD::FNEARBYINT; + break; + case Intrinsic::rint: + ISD = ISD::FRINT; + break; + case Intrinsic::round: + ISD = ISD::FROUND; + break; + case Intrinsic::pow: + ISD = ISD::FPOW; + break; + case Intrinsic::fma: + ISD = ISD::FMA; + break; + case Intrinsic::fmuladd: + ISD = ISD::FMA; + break; + // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free. + case Intrinsic::lifetime_start: + case Intrinsic::lifetime_end: + return 0; + case Intrinsic::masked_store: + return static_cast<T *>(this) + ->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0, 0); + case Intrinsic::masked_load: + return static_cast<T *>(this) + ->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0); + } + + const TargetLoweringBase *TLI = getTLI(); + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(RetTy); + + if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. + // If the type is split to multiple registers, assume that there is some + // overhead to this. + // TODO: Once we have extract/insert subvector cost we need to use them. + if (LT.first > 1) + return LT.first * 2; + return LT.first * 1; + } + + if (!TLI->isOperationExpand(ISD, LT.second)) { + // If the operation is custom lowered then assume + // thare the code is twice as expensive. + return LT.first * 2; + } + + // If we can't lower fmuladd into an FMA estimate the cost as a floating + // point mul followed by an add. + if (IID == Intrinsic::fmuladd) + return static_cast<T *>(this) + ->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) + + static_cast<T *>(this) + ->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy); + + // Else, assume that we need to scalarize this intrinsic. For math builtins + // this will emit a costly libcall, adding call overhead and spills. Make it + // very expensive. + if (RetTy->isVectorTy()) { + unsigned ScalarizationCost = getScalarizationOverhead(RetTy, true, false); + unsigned ScalarCalls = RetTy->getVectorNumElements(); + SmallVector<Type *, 4> ScalarTys; + for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { + Type *Ty = Tys[i]; + if (Ty->isVectorTy()) + Ty = Ty->getScalarType(); + ScalarTys.push_back(Ty); + } + unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost( + IID, RetTy->getScalarType(), ScalarTys); + for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { + if (Tys[i]->isVectorTy()) { + ScalarizationCost += getScalarizationOverhead(Tys[i], false, true); + ScalarCalls = std::max(ScalarCalls, Tys[i]->getVectorNumElements()); + } + } + + return ScalarCalls * ScalarCost + ScalarizationCost; + } + + // This is going to be turned into a library call, make it expensive. + return 10; + } + + /// \brief Compute a cost of the given call instruction. + /// + /// Compute the cost of calling function F with return type RetTy and + /// argument types Tys. F might be nullptr, in this case the cost of an + /// arbitrary call with the specified signature will be returned. + /// This is used, for instance, when we estimate call of a vector + /// counterpart of the given function. + /// \param F Called function, might be nullptr. + /// \param RetTy Return value types. + /// \param Tys Argument types. + /// \returns The cost of Call instruction. + unsigned getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys) { + return 10; + } + + unsigned getNumberOfParts(Type *Tp) { + std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Tp); + return LT.first; + } + + unsigned getAddressComputationCost(Type *Ty, bool IsComplex) { return 0; } + + unsigned getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise) { + assert(Ty->isVectorTy() && "Expect a vector type"); + unsigned NumVecElts = Ty->getVectorNumElements(); + unsigned NumReduxLevels = Log2_32(NumVecElts); + unsigned ArithCost = + NumReduxLevels * + static_cast<T *>(this)->getArithmeticInstrCost(Opcode, Ty); + // Assume the pairwise shuffles add a cost. + unsigned ShuffleCost = + NumReduxLevels * (IsPairwise + 1) * + static_cast<T *>(this) + ->getShuffleCost(TTI::SK_ExtractSubvector, Ty, NumVecElts / 2, Ty); + return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true); + } + + /// @} +}; + +/// \brief Concrete BasicTTIImpl that can be used if no further customization +/// is needed. +class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> { + typedef BasicTTIImplBase<BasicTTIImpl> BaseT; + friend class BasicTTIImplBase<BasicTTIImpl>; + + const TargetSubtargetInfo *ST; + const TargetLoweringBase *TLI; + + const TargetSubtargetInfo *getST() const { return ST; } + const TargetLoweringBase *getTLI() const { return TLI; } + +public: + explicit BasicTTIImpl(const TargetMachine *ST, Function &F); + + // Provide value semantics. MSVC requires that we spell all of these out. + BasicTTIImpl(const BasicTTIImpl &Arg) + : BaseT(static_cast<const BaseT &>(Arg)), ST(Arg.ST), TLI(Arg.TLI) {} + BasicTTIImpl(BasicTTIImpl &&Arg) + : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)), + TLI(std::move(Arg.TLI)) {} + BasicTTIImpl &operator=(const BasicTTIImpl &RHS) { + BaseT::operator=(static_cast<const BaseT &>(RHS)); + ST = RHS.ST; + TLI = RHS.TLI; + return *this; + } + BasicTTIImpl &operator=(BasicTTIImpl &&RHS) { + BaseT::operator=(std::move(static_cast<BaseT &>(RHS))); + ST = std::move(RHS.ST); + TLI = std::move(RHS.TLI); + return *this; + } +}; + +} + +#endif diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index dd7703b1dbf75..1fd4eeb46b38a 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/CallingConv.h" +#include "llvm/MC/MCRegisterInfo.h" #include "llvm/Target/TargetCallingConv.h" namespace llvm { @@ -122,8 +123,8 @@ public: // There is no need to differentiate between a pending CCValAssign and other // kinds, as they are stored in a different list. static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, - LocInfo HTP) { - return getReg(ValNo, ValVT, 0, LocVT, HTP); + LocInfo HTP, unsigned ExtraInfo = 0) { + return getReg(ValNo, ValVT, ExtraInfo, LocVT, HTP); } void convertToReg(unsigned RegNo) { @@ -146,6 +147,7 @@ public: unsigned getLocReg() const { assert(isRegLoc()); return Loc; } unsigned getLocMemOffset() const { assert(isMemLoc()); return Loc; } + unsigned getExtraInfo() const { return Loc; } MVT getLocVT() const { return LocVT; } LocInfo getLocInfo() const { return HTP; } @@ -312,13 +314,13 @@ public: /// produce a single value. void AnalyzeCallResult(MVT VT, CCAssignFn Fn); - /// getFirstUnallocated - Return the first unallocated register in the set, or - /// NumRegs if they are all allocated. - unsigned getFirstUnallocated(const MCPhysReg *Regs, unsigned NumRegs) const { - for (unsigned i = 0; i != NumRegs; ++i) + /// getFirstUnallocated - Return the index of the first unallocated register + /// in the set, or Regs.size() if they are all allocated. + unsigned getFirstUnallocated(ArrayRef<MCPhysReg> Regs) const { + for (unsigned i = 0; i < Regs.size(); ++i) if (!isAllocated(Regs[i])) return i; - return NumRegs; + return Regs.size(); } /// AllocateReg - Attempt to allocate one register. If it is not available, @@ -341,9 +343,9 @@ public: /// AllocateReg - Attempt to allocate one of the specified registers. If none /// are available, return zero. Otherwise, return the first one available, /// marking it and any aliases as allocated. - unsigned AllocateReg(const MCPhysReg *Regs, unsigned NumRegs) { - unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs); - if (FirstUnalloc == NumRegs) + unsigned AllocateReg(ArrayRef<MCPhysReg> Regs) { + unsigned FirstUnalloc = getFirstUnallocated(Regs); + if (FirstUnalloc == Regs.size()) return 0; // Didn't find the reg. // Mark the register and any aliases as allocated. @@ -382,10 +384,9 @@ public: } /// Version of AllocateReg with list of registers to be shadowed. - unsigned AllocateReg(const MCPhysReg *Regs, const MCPhysReg *ShadowRegs, - unsigned NumRegs) { - unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs); - if (FirstUnalloc == NumRegs) + unsigned AllocateReg(ArrayRef<MCPhysReg> Regs, const MCPhysReg *ShadowRegs) { + unsigned FirstUnalloc = getFirstUnallocated(Regs); + if (FirstUnalloc == Regs.size()) return 0; // Didn't find the reg. // Mark the register and any aliases as allocated. @@ -415,8 +416,8 @@ public: /// Version of AllocateStack with list of extra registers to be shadowed. /// Note that, unlike AllocateReg, this shadows ALL of the shadow registers. unsigned AllocateStack(unsigned Size, unsigned Align, - const MCPhysReg *ShadowRegs, unsigned NumShadowRegs) { - for (unsigned i = 0; i < NumShadowRegs; ++i) + ArrayRef<MCPhysReg> ShadowRegs) { + for (unsigned i = 0; i < ShadowRegs.size(); ++i) MarkAllocated(ShadowRegs[i]); return AllocateStack(Size, Align); } diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h index 973c5954f9ad3..a1b9b4e566a26 100644 --- a/include/llvm/CodeGen/CommandFlags.h +++ b/include/llvm/CodeGen/CommandFlags.h @@ -16,9 +16,12 @@ #ifndef LLVM_CODEGEN_COMMANDFLAGS_H #define LLVM_CODEGEN_COMMANDFLAGS_H +#include "llvm/IR/Module.h" #include "llvm/MC/MCTargetOptionsCommandFlags.h" +#include "llvm//MC/SubtargetFeature.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Host.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include <string> @@ -123,11 +126,6 @@ EnableHonorSignDependentRoundingFPMath("enable-sign-dependent-rounding-fp-math", cl::desc("Force codegen to assume rounding mode can change dynamically"), cl::init(false)); -cl::opt<bool> -GenerateSoftFloatCalls("soft-float", - cl::desc("Generate software floating point library calls"), - cl::init(false)); - cl::opt<llvm::FloatABI::ABIType> FloatABIForCalls("float-abi", cl::desc("Choose float ABI type"), @@ -207,6 +205,10 @@ FunctionSections("function-sections", cl::desc("Emit functions into separate sections"), cl::init(false)); +cl::opt<bool> UniqueSectionNames("unique-section-names", + cl::desc("Give unique names to every section"), + cl::init(true)); + cl::opt<llvm::JumpTable::JumpTableType> JTableType("jump-table-type", cl::desc("Choose the type of Jump-Instruction Table for jumptable."), @@ -222,57 +224,17 @@ JTableType("jump-table-type", "Create one table per unique function type."), clEnumValEnd)); -cl::opt<bool> -FCFI("fcfi", - cl::desc("Apply forward-edge control-flow integrity"), - cl::init(false)); - -cl::opt<llvm::CFIntegrity> -CFIType("cfi-type", - cl::desc("Choose the type of Control-Flow Integrity check to add"), - cl::init(CFIntegrity::Sub), - cl::values( - clEnumValN(CFIntegrity::Sub, "sub", - "Subtract the pointer from the table base, then mask."), - clEnumValN(CFIntegrity::Ror, "ror", - "Use rotate to check the offset from a table base."), - clEnumValN(CFIntegrity::Add, "add", - "Mask out the high bits and add to an aligned base."), - clEnumValEnd)); - -cl::opt<bool> -CFIEnforcing("cfi-enforcing", - cl::desc("Enforce CFI or pass the violation to a function."), - cl::init(false)); - -// Note that this option is linked to the cfi-enforcing option above: if -// cfi-enforcing is set, then the cfi-func-name option is entirely ignored. If -// cfi-enforcing is false and no cfi-func-name is set, then a default function -// will be generated that ignores all CFI violations. The expected signature for -// functions called with CFI violations is -// -// void (i8*, i8*) -// -// The first pointer is a C string containing the name of the function in which -// the violation occurs, and the second pointer is the pointer that violated -// CFI. -cl::opt<std::string> -CFIFuncName("cfi-func-name", cl::desc("The name of the CFI function to call"), - cl::init("")); - // Common utility function tightly tied to the options listed here. Initializes // a TargetOptions object with CodeGen flags and returns it. static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { TargetOptions Options; Options.LessPreciseFPMADOption = EnableFPMAD; - Options.NoFramePointerElim = DisableFPElim; Options.AllowFPOpFusion = FuseFPOps; Options.UnsafeFPMath = EnableUnsafeFPMath; Options.NoInfsFPMath = EnableNoInfsFPMath; Options.NoNaNsFPMath = EnableNoNaNsFPMath; Options.HonorSignDependentRoundingFPMathOption = EnableHonorSignDependentRoundingFPMath; - Options.UseSoftFloat = GenerateSoftFloatCalls; if (FloatABIForCalls != FloatABI::Default) Options.FloatABIType = FloatABIForCalls; Options.NoZerosInBSS = DontPlaceZerosInBSS; @@ -284,17 +246,71 @@ static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.UseInitArray = !UseCtors; Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; + Options.UniqueSectionNames = UniqueSectionNames; Options.MCOptions = InitMCTargetOptionsFromFlags(); Options.JTType = JTableType; - Options.FCFI = FCFI; - Options.CFIType = CFIType; - Options.CFIEnforcing = CFIEnforcing; - Options.CFIFuncName = CFIFuncName; Options.ThreadModel = TMModel; return Options; } +static inline std::string getCPUStr() { + // If user asked for the 'native' CPU, autodetect here. If autodection fails, + // this will set the CPU to an empty string which tells the target to + // pick a basic default. + if (MCPU == "native") + return sys::getHostCPUName(); + + return MCPU; +} + +static inline std::string getFeaturesStr() { + SubtargetFeatures Features; + + // If user asked for the 'native' CPU, we need to autodetect features. + // This is necessary for x86 where the CPU might not support all the + // features the autodetected CPU name lists in the target. For example, + // not all Sandybridge processors support AVX. + if (MCPU == "native") { + StringMap<bool> HostFeatures; + if (sys::getHostCPUFeatures(HostFeatures)) + for (auto &F : HostFeatures) + Features.AddFeature(F.first(), F.second); + } + + for (unsigned i = 0; i != MAttrs.size(); ++i) + Features.AddFeature(MAttrs[i]); + + return Features.getString(); +} + +/// \brief Set function attributes of functions in Module M based on CPU, +/// Features, and command line flags. +static inline void setFunctionAttributes(StringRef CPU, StringRef Features, + Module &M) { + for (auto &F : M) { + auto &Ctx = F.getContext(); + AttributeSet Attrs = F.getAttributes(), NewAttrs; + + if (!CPU.empty()) + NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex, + "target-cpu", CPU); + + if (!Features.empty()) + NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex, + "target-features", Features); + + if (DisableFPElim.getNumOccurrences() > 0) + NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex, + "no-frame-pointer-elim", + DisableFPElim ? "true" : "false"); + + // Let NewAttrs override Attrs. + NewAttrs = Attrs.addAttributes(Ctx, AttributeSet::FunctionIndex, NewAttrs); + F.setAttributes(NewAttrs); + } +} + #endif diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h index e310aef3dcbbb..8e40ef77d374f 100644 --- a/include/llvm/CodeGen/DIE.h +++ b/include/llvm/CodeGen/DIE.h @@ -16,6 +16,7 @@ #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/DwarfStringPoolEntry.h" #include "llvm/Support/Dwarf.h" #include <vector> @@ -95,7 +96,7 @@ public: /// Emit - Print the abbreviation using the specified asm printer. /// - void Emit(AsmPrinter *AP) const; + void Emit(const AsmPrinter *AP) const; #ifndef NDEBUG void print(raw_ostream &O); @@ -200,8 +201,6 @@ public: /// to DWARF attribute classes. /// class DIEValue { - virtual void anchor(); - public: enum Type { isInteger, @@ -216,13 +215,14 @@ public: isLocList, }; -protected: +private: /// Ty - Type of data stored in the value. /// Type Ty; +protected: explicit DIEValue(Type T) : Ty(T) {} - virtual ~DIEValue() {} + ~DIEValue() {} public: // Accessors @@ -230,14 +230,14 @@ public: /// EmitValue - Emit value via the Dwarf writer. /// - virtual void EmitValue(AsmPrinter *AP, dwarf::Form Form) const = 0; + void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; /// SizeOf - Return the size of a value in bytes. /// - virtual unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const = 0; + unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; #ifndef NDEBUG - virtual void print(raw_ostream &O) const = 0; + void print(raw_ostream &O) const; void dump() const; #endif }; @@ -246,6 +246,8 @@ public: /// DIEInteger - An integer value DIE. /// class DIEInteger : public DIEValue { + friend DIEValue; + uint64_t Integer; public: @@ -273,21 +275,18 @@ public: return dwarf::DW_FORM_data8; } - /// EmitValue - Emit integer of appropriate size. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - uint64_t getValue() const { return Integer; } - - /// SizeOf - Determine size of integer value in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; + void setValue(uint64_t Val) { Integer = Val; } // Implement isa/cast/dyncast. static bool classof(const DIEValue *I) { return I->getType() == isInteger; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -295,28 +294,26 @@ public: /// DIEExpr - An expression DIE. // class DIEExpr : public DIEValue { + friend class DIEValue; + const MCExpr *Expr; public: explicit DIEExpr(const MCExpr *E) : DIEValue(isExpr), Expr(E) {} - /// EmitValue - Emit expression value. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - /// getValue - Get MCExpr. /// const MCExpr *getValue() const { return Expr; } - /// SizeOf - Determine size of expression value in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *E) { return E->getType() == isExpr; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -324,28 +321,26 @@ public: /// DIELabel - A label DIE. // class DIELabel : public DIEValue { + friend class DIEValue; + const MCSymbol *Label; public: explicit DIELabel(const MCSymbol *L) : DIEValue(isLabel), Label(L) {} - /// EmitValue - Emit label value. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - /// getValue - Get MCSymbol. /// const MCSymbol *getValue() const { return Label; } - /// SizeOf - Determine size of label value in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *L) { return L->getType() == isLabel; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -353,6 +348,8 @@ public: /// DIEDelta - A simple label difference DIE. /// class DIEDelta : public DIEValue { + friend class DIEValue; + const MCSymbol *LabelHi; const MCSymbol *LabelLo; @@ -360,19 +357,15 @@ public: DIEDelta(const MCSymbol *Hi, const MCSymbol *Lo) : DIEValue(isDelta), LabelHi(Hi), LabelLo(Lo) {} - /// EmitValue - Emit delta value. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of delta value in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *D) { return D->getType() == isDelta; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -380,29 +373,25 @@ public: /// DIEString - A container for string values. /// class DIEString : public DIEValue { - const DIEValue *Access; - StringRef Str; + friend class DIEValue; + + DwarfStringPoolEntryRef S; public: - DIEString(const DIEValue *Acc, StringRef S) - : DIEValue(isString), Access(Acc), Str(S) {} + DIEString(DwarfStringPoolEntryRef S) : DIEValue(isString), S(S) {} /// getString - Grab the string out of the object. - StringRef getString() const { return Str; } - - /// EmitValue - Emit delta value. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of delta value in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; + StringRef getString() const { return S.getString(); } // Implement isa/cast/dyncast. static bool classof(const DIEValue *D) { return D->getType() == isString; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -411,6 +400,8 @@ public: /// this class can also be used as a proxy for a debug information entry not /// yet defined (ie. types.) class DIEEntry : public DIEValue { + friend class DIEValue; + DIE &Entry; public: @@ -419,53 +410,49 @@ public: DIE &getEntry() const { return Entry; } - /// EmitValue - Emit debug information entry offset. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of debug information entry in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override { - return Form == dwarf::DW_FORM_ref_addr ? getRefAddrSize(AP) - : sizeof(int32_t); - } - /// Returns size of a ref_addr entry. - static unsigned getRefAddrSize(AsmPrinter *AP); + static unsigned getRefAddrSize(const AsmPrinter *AP); // Implement isa/cast/dyncast. static bool classof(const DIEValue *E) { return E->getType() == isEntry; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const { + return Form == dwarf::DW_FORM_ref_addr ? getRefAddrSize(AP) + : sizeof(int32_t); + } + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; //===--------------------------------------------------------------------===// /// \brief A signature reference to a type unit. class DIETypeSignature : public DIEValue { + friend class DIEValue; + const DwarfTypeUnit &Unit; public: explicit DIETypeSignature(const DwarfTypeUnit &Unit) : DIEValue(isTypeSignature), Unit(Unit) {} - /// \brief Emit type unit signature. - void EmitValue(AsmPrinter *Asm, dwarf::Form Form) const override; + // \brief Implement isa/cast/dyncast. + static bool classof(const DIEValue *E) { + return E->getType() == isTypeSignature; + } - /// Returns size of a ref_sig8 entry. - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override { +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const { assert(Form == dwarf::DW_FORM_ref_sig8); return 8; } - // \brief Implement isa/cast/dyncast. - static bool classof(const DIEValue *E) { - return E->getType() == isTypeSignature; - } #ifndef NDEBUG - void print(raw_ostream &O) const override; - void dump() const; + void printImpl(raw_ostream &O) const; #endif }; @@ -473,13 +460,15 @@ public: /// DIELoc - Represents an expression location. // class DIELoc : public DIEValue, public DIE { + friend class DIEValue; + mutable unsigned Size; // Size in bytes excluding size header. public: DIELoc() : DIEValue(isLoc), Size(0) {} /// ComputeSize - Calculate the size of the location expression. /// - unsigned ComputeSize(AsmPrinter *AP) const; + unsigned ComputeSize(const AsmPrinter *AP) const; /// BestForm - Choose the best form for data. /// @@ -496,19 +485,15 @@ public: return dwarf::DW_FORM_block; } - /// EmitValue - Emit location data. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of location data in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *E) { return E->getType() == isLoc; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -516,13 +501,15 @@ public: /// DIEBlock - Represents a block of values. // class DIEBlock : public DIEValue, public DIE { + friend class DIEValue; + mutable unsigned Size; // Size in bytes excluding size header. public: DIEBlock() : DIEValue(isBlock), Size(0) {} /// ComputeSize - Calculate the size of the location expression. /// - unsigned ComputeSize(AsmPrinter *AP) const; + unsigned ComputeSize(const AsmPrinter *AP) const; /// BestForm - Choose the best form for data. /// @@ -536,19 +523,15 @@ public: return dwarf::DW_FORM_block; } - /// EmitValue - Emit location data. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of location data in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *E) { return E->getType() == isBlock; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; @@ -557,6 +540,8 @@ public: /// section. // class DIELocList : public DIEValue { + friend class DIEValue; + // Index into the .debug_loc vector. size_t Index; @@ -566,19 +551,15 @@ public: /// getValue - Grab the current index out. size_t getValue() const { return Index; } - /// EmitValue - Emit location data. - /// - void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override; - - /// SizeOf - Determine size of location data in bytes. - /// - unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override; - // Implement isa/cast/dyncast. static bool classof(const DIEValue *E) { return E->getType() == isLocList; } +private: + void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const; + #ifndef NDEBUG - void print(raw_ostream &O) const override; + void printImpl(raw_ostream &O) const; #endif }; diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h new file mode 100644 index 0000000000000..fc2b5ddd2d2cf --- /dev/null +++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h @@ -0,0 +1,51 @@ +//===- llvm/CodeGen/DwarfStringPoolEntry.h - String pool entry --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H +#define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H + +#include "llvm/ADT/StringMap.h" + +namespace llvm { + +class MCSymbol; + +/// Data for a string pool entry. +struct DwarfStringPoolEntry { + MCSymbol *Symbol; + unsigned Offset; + unsigned Index; +}; + +/// String pool entry reference. +struct DwarfStringPoolEntryRef { + const StringMapEntry<DwarfStringPoolEntry> *I = nullptr; + +public: + DwarfStringPoolEntryRef() = default; + explicit DwarfStringPoolEntryRef( + const StringMapEntry<DwarfStringPoolEntry> &I) + : I(&I) {} + + explicit operator bool() const { return I; } + MCSymbol *getSymbol() const { + assert(I->second.Symbol && "No symbol available!"); + return I->second.Symbol; + } + unsigned getOffset() const { return I->second.Offset; } + unsigned getIndex() const { return I->second.Index; } + StringRef getString() const { return I->first(); } + + bool operator==(const DwarfStringPoolEntryRef &X) const { return I == X.I; } + bool operator!=(const DwarfStringPoolEntryRef &X) const { return I != X.I; } +}; + +} // end namespace llvm + +#endif diff --git a/include/llvm/CodeGen/ForwardControlFlowIntegrity.h b/include/llvm/CodeGen/ForwardControlFlowIntegrity.h deleted file mode 100644 index ec8e2ef243b7d..0000000000000 --- a/include/llvm/CodeGen/ForwardControlFlowIntegrity.h +++ /dev/null @@ -1,122 +0,0 @@ -//===-- ForwardControlFlowIntegrity.h: Forward-Edge CFI ---------*- C++ -*-===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass instruments indirect calls with checks to ensure that these calls -// pass through the appropriate jump-instruction table generated by -// JumpInstrTables. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H -#define LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Pass.h" -#include "llvm/Target/TargetOptions.h" -#include <string> - -namespace llvm { - -class AnalysisUsage; -class BasicBlock; -class Constant; -class Function; -class Instruction; -class Module; -class Value; - -/// ForwardControlFlowIntegrity uses the information from JumpInstrTableInfo to -/// prepend checks to indirect calls to make sure that these calls target valid -/// locations. -class ForwardControlFlowIntegrity : public ModulePass { -public: - static char ID; - - ForwardControlFlowIntegrity(); - ForwardControlFlowIntegrity(JumpTable::JumpTableType JTT, - CFIntegrity CFIType, - bool CFIEnforcing, std::string CFIFuncName); - ~ForwardControlFlowIntegrity() override; - - /// Runs the CFI pass on a given module. This works best if the module in - /// question is the result of link-time optimization (see lib/LTO). - bool runOnModule(Module &M) override; - const char *getPassName() const override { - return "Forward Control-Flow Integrity"; - } - void getAnalysisUsage(AnalysisUsage &AU) const override; - -private: - typedef SmallVector<Instruction *, 64> CallSet; - - /// A structure that is used to keep track of constant table information. - struct CFIConstants { - Constant *StartValue; - Constant *MaskValue; - Constant *Size; - }; - - /// A map from function type to the base of the table for this type and a mask - /// for the table - typedef DenseMap<FunctionType *, CFIConstants> CFITables; - - CallSet IndirectCalls; - - /// The type of jumptable implementation. - JumpTable::JumpTableType JTType; - - /// The type of CFI check to add before each indirect call. - CFIntegrity CFIType; - - /// A value that controls whether or not CFI violations cause a halt. - bool CFIEnforcing; - - /// The name of the function to call in case of a CFI violation when - /// CFIEnforcing is false. There is a default function that ignores - /// violations. - std::string CFIFuncName; - - /// The alignment of each entry in the table, from JumpInstrTableInfo. The - /// JumpInstrTableInfo class always makes this a power of two. - uint64_t ByteAlignment; - - /// The base-2 logarithm of ByteAlignment, needed for some of the transforms - /// (like CFIntegrity::Ror) - unsigned LogByteAlignment; - - /// Adds checks to each indirect call site to make sure that it is calling a - /// function in our jump table. - void updateIndirectCalls(Module &M, CFITables &CFIT); - - /// Walks the instructions to find all the indirect calls. - void getIndirectCalls(Module &M); - - /// Adds a function that handles violations in non-enforcing mode - /// (!CFIEnforcing). The default warning function simply returns, since the - /// exact details of how to handle CFI violations depend on the application. - void addWarningFunction(Module &M); - - /// Rewrites a function pointer in a call/invoke instruction to force it into - /// a table. - void rewriteFunctionPointer(Module &M, Instruction *I, Value *FunPtr, - Constant *JumpTableStart, Constant *JumpTableMask, - Constant *JumpTableSize); - - /// Inserts a check and a call to a warning function at a given instruction - /// that must be an indirect call. - void insertWarning(Module &M, BasicBlock *Block, Instruction *I, - Value *FunPtr); -}; - -ModulePass * -createForwardControlFlowIntegrityPass(JumpTable::JumpTableType JTT, - CFIntegrity CFIType, - bool CFIEnforcing, StringRef CFIFuncName); -} - -#endif // LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index 7c574df4ba414..82c762ed850fe 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -18,6 +18,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/ISDOpcodes.h" @@ -73,6 +74,16 @@ public: /// cross-basic-block values. DenseMap<const Value*, unsigned> ValueMap; + // Keep track of frame indices allocated for statepoints as they could be used + // across basic block boundaries. + // Key of the map is statepoint instruction, value is a map from spilled + // llvm Value to the optional stack stack slot index. + // If optional is unspecified it means that we have visited this value + // but didn't spill it. + typedef DenseMap<const Value*, Optional<int>> StatepointSpilledValueMapTy; + DenseMap<const Instruction*, StatepointSpilledValueMapTy> + StatepointRelocatedValues; + /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in /// the entry block. This allows the allocas to be efficiently referenced /// anywhere in the function. @@ -221,6 +232,8 @@ public: int getArgumentFrameIndex(const Argument *A); private: + void addSEHHandlersForLPads(ArrayRef<const LandingPadInst *> LPads); + /// LiveOutRegInfo - Information about live out vregs. IndexedMap<LiveOutInfo, VirtReg2IndexFunctor> LiveOutRegInfo; }; @@ -232,11 +245,6 @@ private: /// floating-point support. void ComputeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo *MMI); -/// AddCatchInfo - Extract the personality and type infos from an eh.selector -/// call, and add them to the specified machine basic block. -void AddCatchInfo(const CallInst &I, - MachineModuleInfo *MMI, MachineBasicBlock *MBB); - /// AddLandingPadInfo - Extract the exception handling information from the /// landingpad instruction and add them to the specified machine module info. void AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI, diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h index c7f1ab87fcb11..357b2d8a7ca7a 100644 --- a/include/llvm/CodeGen/GCMetadata.h +++ b/include/llvm/CodeGen/GCMetadata.h @@ -34,184 +34,173 @@ #define LLVM_CODEGEN_GCMETADATA_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CodeGen/GCStrategy.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" #include <memory> namespace llvm { - class AsmPrinter; - class GCStrategy; - class Constant; - class MCSymbol; - - namespace GC { - /// PointKind - The type of a collector-safe point. - /// - enum PointKind { - Loop, ///< Instr is a loop (backwards branch). - Return, ///< Instr is a return instruction. - PreCall, ///< Instr is a call instruction. - PostCall ///< Instr is the return address of a call. - }; +class AsmPrinter; +class Constant; +class MCSymbol; + +/// GCPoint - Metadata for a collector-safe point in machine code. +/// +struct GCPoint { + GC::PointKind Kind; ///< The kind of the safe point. + MCSymbol *Label; ///< A label. + DebugLoc Loc; + + GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL) + : Kind(K), Label(L), Loc(DL) {} +}; + +/// GCRoot - Metadata for a pointer to an object managed by the garbage +/// collector. +struct GCRoot { + int Num; ///< Usually a frame index. + int StackOffset; ///< Offset from the stack pointer. + const Constant *Metadata; ///< Metadata straight from the call + ///< to llvm.gcroot. + + GCRoot(int N, const Constant *MD) : Num(N), StackOffset(-1), Metadata(MD) {} +}; + +/// Garbage collection metadata for a single function. Currently, this +/// information only applies to GCStrategies which use GCRoot. +class GCFunctionInfo { +public: + typedef std::vector<GCPoint>::iterator iterator; + typedef std::vector<GCRoot>::iterator roots_iterator; + typedef std::vector<GCRoot>::const_iterator live_iterator; + +private: + const Function &F; + GCStrategy &S; + uint64_t FrameSize; + std::vector<GCRoot> Roots; + std::vector<GCPoint> SafePoints; + + // FIXME: Liveness. A 2D BitVector, perhaps? + // + // BitVector Liveness; + // + // bool islive(int point, int root) = + // Liveness[point * SafePoints.size() + root] + // + // The bit vector is the more compact representation where >3.2% of roots + // are live per safe point (1.5% on 64-bit hosts). + +public: + GCFunctionInfo(const Function &F, GCStrategy &S); + ~GCFunctionInfo(); + + /// getFunction - Return the function to which this metadata applies. + /// + const Function &getFunction() const { return F; } + + /// getStrategy - Return the GC strategy for the function. + /// + GCStrategy &getStrategy() { return S; } + + /// addStackRoot - Registers a root that lives on the stack. Num is the + /// stack object ID for the alloca (if the code generator is + // using MachineFrameInfo). + void addStackRoot(int Num, const Constant *Metadata) { + Roots.push_back(GCRoot(Num, Metadata)); + } + + /// removeStackRoot - Removes a root. + roots_iterator removeStackRoot(roots_iterator position) { + return Roots.erase(position); + } + + /// addSafePoint - Notes the existence of a safe point. Num is the ID of the + /// label just prior to the safe point (if the code generator is using + /// MachineModuleInfo). + void addSafePoint(GC::PointKind Kind, MCSymbol *Label, DebugLoc DL) { + SafePoints.push_back(GCPoint(Kind, Label, DL)); } - /// GCPoint - Metadata for a collector-safe point in machine code. + /// getFrameSize/setFrameSize - Records the function's frame size. /// - struct GCPoint { - GC::PointKind Kind; ///< The kind of the safe point. - MCSymbol *Label; ///< A label. - DebugLoc Loc; - - GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL) - : Kind(K), Label(L), Loc(DL) {} - }; - - /// GCRoot - Metadata for a pointer to an object managed by the garbage - /// collector. - struct GCRoot { - int Num; ///< Usually a frame index. - int StackOffset; ///< Offset from the stack pointer. - const Constant *Metadata; ///< Metadata straight from the call - ///< to llvm.gcroot. - - GCRoot(int N, const Constant *MD) : Num(N), StackOffset(-1), Metadata(MD) {} - }; - - - /// Garbage collection metadata for a single function. Currently, this - /// information only applies to GCStrategies which use GCRoot. - class GCFunctionInfo { - public: - typedef std::vector<GCPoint>::iterator iterator; - typedef std::vector<GCRoot>::iterator roots_iterator; - typedef std::vector<GCRoot>::const_iterator live_iterator; - - private: - const Function &F; - GCStrategy &S; - uint64_t FrameSize; - std::vector<GCRoot> Roots; - std::vector<GCPoint> SafePoints; - - // FIXME: Liveness. A 2D BitVector, perhaps? - // - // BitVector Liveness; - // - // bool islive(int point, int root) = - // Liveness[point * SafePoints.size() + root] - // - // The bit vector is the more compact representation where >3.2% of roots - // are live per safe point (1.5% on 64-bit hosts). - - public: - GCFunctionInfo(const Function &F, GCStrategy &S); - ~GCFunctionInfo(); - - /// getFunction - Return the function to which this metadata applies. - /// - const Function &getFunction() const { return F; } - - /// getStrategy - Return the GC strategy for the function. - /// - GCStrategy &getStrategy() { return S; } - - /// addStackRoot - Registers a root that lives on the stack. Num is the - /// stack object ID for the alloca (if the code generator is - // using MachineFrameInfo). - void addStackRoot(int Num, const Constant *Metadata) { - Roots.push_back(GCRoot(Num, Metadata)); - } - - /// removeStackRoot - Removes a root. - roots_iterator removeStackRoot(roots_iterator position) { - return Roots.erase(position); - } - - /// addSafePoint - Notes the existence of a safe point. Num is the ID of the - /// label just prior to the safe point (if the code generator is using - /// MachineModuleInfo). - void addSafePoint(GC::PointKind Kind, MCSymbol *Label, DebugLoc DL) { - SafePoints.push_back(GCPoint(Kind, Label, DL)); - } - - /// getFrameSize/setFrameSize - Records the function's frame size. - /// - uint64_t getFrameSize() const { return FrameSize; } - void setFrameSize(uint64_t S) { FrameSize = S; } - - /// begin/end - Iterators for safe points. - /// - iterator begin() { return SafePoints.begin(); } - iterator end() { return SafePoints.end(); } - size_t size() const { return SafePoints.size(); } - - /// roots_begin/roots_end - Iterators for all roots in the function. - /// - roots_iterator roots_begin() { return Roots.begin(); } - roots_iterator roots_end () { return Roots.end(); } - size_t roots_size() const { return Roots.size(); } - - /// live_begin/live_end - Iterators for live roots at a given safe point. - /// - live_iterator live_begin(const iterator &p) { return roots_begin(); } - live_iterator live_end (const iterator &p) { return roots_end(); } - size_t live_size(const iterator &p) const { return roots_size(); } - }; - - /// An analysis pass which caches information about the entire Module. - /// Records both the function level information used by GCRoots and a - /// cache of the 'active' gc strategy objects for the current Module. - class GCModuleInfo : public ImmutablePass { - typedef StringMap<GCStrategy*> strategy_map_type; - typedef std::vector<std::unique_ptr<GCStrategy>> list_type; - - strategy_map_type StrategyMap; - list_type StrategyList; - - GCStrategy *getOrCreateStrategy(const Module *M, const std::string &Name); - - public: - /// List of per function info objects. In theory, Each of these - /// may be associated with a different GC. - typedef std::vector<std::unique_ptr<GCFunctionInfo>> FuncInfoVec; - - FuncInfoVec::iterator funcinfo_begin() { return Functions.begin(); } - FuncInfoVec::iterator funcinfo_end() { return Functions.end(); } - - - private: - /// Owning list of all GCFunctionInfos associated with this Module - FuncInfoVec Functions; - - /// Non-owning map to bypass linear search when finding the GCFunctionInfo - /// associated with a particular Function. - typedef DenseMap<const Function*,GCFunctionInfo*> finfo_map_type; - finfo_map_type FInfoMap; - public: - - typedef list_type::const_iterator iterator; - - static char ID; - - GCModuleInfo(); - - /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should - /// call it in doFinalization(). - /// - void clear(); - - /// begin/end - Iterators for used strategies. - /// - iterator begin() const { return StrategyList.begin(); } - iterator end() const { return StrategyList.end(); } - - /// get - Look up function metadata. This is currently assumed - /// have the side effect of initializing the associated GCStrategy. That - /// will soon change. - GCFunctionInfo &getFunctionInfo(const Function &F); - }; + uint64_t getFrameSize() const { return FrameSize; } + void setFrameSize(uint64_t S) { FrameSize = S; } + /// begin/end - Iterators for safe points. + /// + iterator begin() { return SafePoints.begin(); } + iterator end() { return SafePoints.end(); } + size_t size() const { return SafePoints.size(); } + + /// roots_begin/roots_end - Iterators for all roots in the function. + /// + roots_iterator roots_begin() { return Roots.begin(); } + roots_iterator roots_end() { return Roots.end(); } + size_t roots_size() const { return Roots.size(); } + + /// live_begin/live_end - Iterators for live roots at a given safe point. + /// + live_iterator live_begin(const iterator &p) { return roots_begin(); } + live_iterator live_end(const iterator &p) { return roots_end(); } + size_t live_size(const iterator &p) const { return roots_size(); } +}; + +/// An analysis pass which caches information about the entire Module. +/// Records both the function level information used by GCRoots and a +/// cache of the 'active' gc strategy objects for the current Module. +class GCModuleInfo : public ImmutablePass { + /// An owning list of all GCStrategies which have been created + SmallVector<std::unique_ptr<GCStrategy>, 1> GCStrategyList; + /// A helper map to speedup lookups into the above list + StringMap<GCStrategy*> GCStrategyMap; + +public: + /// Lookup the GCStrategy object associated with the given gc name. + /// Objects are owned internally; No caller should attempt to delete the + /// returned objects. + GCStrategy *getGCStrategy(const StringRef Name); + + /// List of per function info objects. In theory, Each of these + /// may be associated with a different GC. + typedef std::vector<std::unique_ptr<GCFunctionInfo>> FuncInfoVec; + + FuncInfoVec::iterator funcinfo_begin() { return Functions.begin(); } + FuncInfoVec::iterator funcinfo_end() { return Functions.end(); } + +private: + /// Owning list of all GCFunctionInfos associated with this Module + FuncInfoVec Functions; + + /// Non-owning map to bypass linear search when finding the GCFunctionInfo + /// associated with a particular Function. + typedef DenseMap<const Function *, GCFunctionInfo *> finfo_map_type; + finfo_map_type FInfoMap; + +public: + typedef SmallVector<std::unique_ptr<GCStrategy>,1>::const_iterator iterator; + + static char ID; + + GCModuleInfo(); + + /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should + /// call it in doFinalization(). + /// + void clear(); + + /// begin/end - Iterators for used strategies. + /// + iterator begin() const { return GCStrategyList.begin(); } + iterator end() const { return GCStrategyList.end(); } + + /// get - Look up function metadata. This is currently assumed + /// have the side effect of initializing the associated GCStrategy. That + /// will soon change. + GCFunctionInfo &getFunctionInfo(const Function &F); +}; } #endif diff --git a/include/llvm/CodeGen/GCMetadataPrinter.h b/include/llvm/CodeGen/GCMetadataPrinter.h index 25fafba93f8b6..2208470291138 100644 --- a/include/llvm/CodeGen/GCMetadataPrinter.h +++ b/include/llvm/CodeGen/GCMetadataPrinter.h @@ -26,43 +26,39 @@ namespace llvm { - class GCMetadataPrinter; +class GCMetadataPrinter; - /// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the - /// defaults from Registry. - typedef Registry<GCMetadataPrinter> GCMetadataPrinterRegistry; +/// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the +/// defaults from Registry. +typedef Registry<GCMetadataPrinter> GCMetadataPrinterRegistry; - /// GCMetadataPrinter - Emits GC metadata as assembly code. Instances are - /// created, managed, and owned by the AsmPrinter. - class GCMetadataPrinter { - private: - GCStrategy *S; - friend class AsmPrinter; +/// GCMetadataPrinter - Emits GC metadata as assembly code. Instances are +/// created, managed, and owned by the AsmPrinter. +class GCMetadataPrinter { +private: + GCStrategy *S; + friend class AsmPrinter; - protected: - // May only be subclassed. - GCMetadataPrinter(); +protected: + // May only be subclassed. + GCMetadataPrinter(); - private: - GCMetadataPrinter(const GCMetadataPrinter &) LLVM_DELETED_FUNCTION; - GCMetadataPrinter & - operator=(const GCMetadataPrinter &) LLVM_DELETED_FUNCTION; +private: + GCMetadataPrinter(const GCMetadataPrinter &) = delete; + GCMetadataPrinter &operator=(const GCMetadataPrinter &) = delete; - public: - GCStrategy &getStrategy() { return *S; } +public: + GCStrategy &getStrategy() { return *S; } - /// Called before the assembly for the module is generated by - /// the AsmPrinter (but after target specific hooks.) - virtual void beginAssembly(Module &M, GCModuleInfo &Info, - AsmPrinter &AP) {} - /// Called after the assembly for the module is generated by - /// the AsmPrinter (but before target specific hooks) - virtual void finishAssembly(Module &M, GCModuleInfo &Info, - AsmPrinter &AP) {} - - virtual ~GCMetadataPrinter(); - }; + /// Called before the assembly for the module is generated by + /// the AsmPrinter (but after target specific hooks.) + virtual void beginAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {} + /// Called after the assembly for the module is generated by + /// the AsmPrinter (but before target specific hooks) + virtual void finishAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {} + virtual ~GCMetadataPrinter(); +}; } #endif diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h index 0b0c3124c537c..a1b8e895898fe 100644 --- a/include/llvm/CodeGen/GCStrategy.h +++ b/include/llvm/CodeGen/GCStrategy.h @@ -16,11 +16,11 @@ // GCStrategy is relevant for implementations using either gc.root or // gc.statepoint based lowering strategies, but is currently focused mostly on // options for gc.root. This will change over time. -// +// // When requested by a subclass of GCStrategy, the gc.root implementation will // populate GCModuleInfo and GCFunctionInfo with that about each Function in // the Module that opts in to garbage collection. Specifically: -// +// // - Safe points // Garbage collection is generally only possible at certain points in code. // GCStrategy can request that the collector insert such points: @@ -28,7 +28,7 @@ // - At and after any call to a subroutine // - Before returning from the current function // - Before backwards branches (loops) -// +// // - Roots // When a reference to a GC-allocated object exists on the stack, it must be // stored in an alloca registered with llvm.gcoot. @@ -42,155 +42,136 @@ // insertion support is planned. gc.statepoint does not currently support // custom stack map formats; such can be generated by parsing the standard // stack map section if desired. -// +// // The read and write barrier support can be used with either implementation. // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GCSTRATEGY_H -#define LLVM_CODEGEN_GCSTRATEGY_H +#ifndef LLVM_IR_GCSTRATEGY_H +#define LLVM_IR_GCSTRATEGY_H #include "llvm/ADT/Optional.h" -#include "llvm/CodeGen/GCMetadata.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Registry.h" #include <string> namespace llvm { - /// GCStrategy describes a garbage collector algorithm's code generation - /// requirements, and provides overridable hooks for those needs which cannot - /// be abstractly described. GCStrategy objects currently must be looked up - /// through the GCModuleInfo analysis pass. They are owned by the analysis - /// pass and recreated every time that pass is invalidated. - class GCStrategy { - private: - std::string Name; - friend class GCModuleInfo; - - protected: - bool UseStatepoints; /// Uses gc.statepoints as opposed to gc.roots, - /// if set, none of the other options can be - /// anything but their default values. - - unsigned NeededSafePoints; ///< Bitmask of required safe points. - bool CustomReadBarriers; ///< Default is to insert loads. - bool CustomWriteBarriers; ///< Default is to insert stores. - bool CustomRoots; ///< Default is to pass through to backend. - bool CustomSafePoints; ///< Default is to use NeededSafePoints - ///< to find safe points. - bool InitRoots; ///< If set, roots are nulled during lowering. - bool UsesMetadata; ///< If set, backend must emit metadata tables. - - public: - GCStrategy(); - virtual ~GCStrategy() {} - - /// Return the name of the GC strategy. This is the value of the collector - /// name string specified on functions which use this strategy. - const std::string &getName() const { return Name; } - - /// By default, write barriers are replaced with simple store - /// instructions. If true, then performCustomLowering must instead lower - /// them. - bool customWriteBarrier() const { return CustomWriteBarriers; } - - /// By default, read barriers are replaced with simple load - /// instructions. If true, then performCustomLowering must instead lower - /// them. - bool customReadBarrier() const { return CustomReadBarriers; } - - /// Returns true if this strategy is expecting the use of gc.statepoints, - /// and false otherwise. - bool useStatepoints() const { return UseStatepoints; } - - /** @name Statepoint Specific Properties */ - ///@{ - - /// If the value specified can be reliably distinguished, returns true for - /// pointers to GC managed locations and false for pointers to non-GC - /// managed locations. Note a GCStrategy can always return 'None' (i.e. an - /// empty optional indicating it can't reliably distinguish. - virtual Optional<bool> isGCManagedPointer(const Value *V) const { - return None; - } - ///@} - - /** @name GCRoot Specific Properties - * These properties and overrides only apply to collector strategies using - * GCRoot. - */ - ///@{ - - /// True if safe points of any kind are required. By default, none are - /// recorded. - bool needsSafePoints() const { - return CustomSafePoints || NeededSafePoints != 0; - } - - /// True if the given kind of safe point is required. By default, none are - /// recorded. - bool needsSafePoint(GC::PointKind Kind) const { - return (NeededSafePoints & 1 << Kind) != 0; - } - - /// By default, roots are left for the code generator so it can generate a - /// stack map. If true, then performCustomLowering must delete them. - bool customRoots() const { return CustomRoots; } - - /// By default, the GC analysis will find safe points according to - /// NeededSafePoints. If true, then findCustomSafePoints must create them. - bool customSafePoints() const { return CustomSafePoints; } - - /// If set, gcroot intrinsics should initialize their allocas to null - /// before the first use. This is necessary for most GCs and is enabled by - /// default. - bool initializeRoots() const { return InitRoots; } - - /// If set, appropriate metadata tables must be emitted by the back-end - /// (assembler, JIT, or otherwise). For statepoint, this method is - /// currently unsupported. The stackmap information can be found in the - /// StackMap section as described in the documentation. - bool usesMetadata() const { return UsesMetadata; } - - ///@} - - /// initializeCustomLowering/performCustomLowering - If any of the actions - /// are set to custom, performCustomLowering must be overriden to transform - /// the corresponding actions to LLVM IR. initializeCustomLowering is - /// optional to override. These are the only GCStrategy methods through - /// which the LLVM IR can be modified. These methods apply mostly to - /// gc.root based implementations, but can be overriden to provide custom - /// barrier lowerings with gc.statepoint as well. - ///@{ - virtual bool initializeCustomLowering(Module &F) { - // No changes made - return false; - } - virtual bool performCustomLowering(Function &F) { - llvm_unreachable("GCStrategy subclass specified a configuration which" - "requires a custom lowering without providing one"); - } - ///@} - /// Called if customSafepoints returns true, used only by gc.root - /// implementations. - virtual bool findCustomSafePoints(GCFunctionInfo& FI, MachineFunction& MF) { - llvm_unreachable("GCStrategy subclass specified a configuration which" - "requests custom safepoint identification without" - "providing an implementation for such"); - } - }; - - /// Subclasses of GCStrategy are made available for use during compilation by - /// adding them to the global GCRegistry. This can done either within the - /// LLVM source tree or via a loadable plugin. An example registeration - /// would be: - /// static GCRegistry::Add<CustomGC> X("custom-name", - /// "my custom supper fancy gc strategy"); - /// - /// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also - /// register your GCMetadataPrinter subclass with the - /// GCMetadataPrinterRegistery as well. - typedef Registry<GCStrategy> GCRegistry; +namespace GC { +/// PointKind - Used to indicate whether the address of the call instruction +/// or the address after the call instruction is listed in the stackmap. For +/// most runtimes, PostCall safepoints are appropriate. +/// +enum PointKind { + PreCall, ///< Instr is a call instruction. + PostCall ///< Instr is the return address of a call. +}; +} + +/// GCStrategy describes a garbage collector algorithm's code generation +/// requirements, and provides overridable hooks for those needs which cannot +/// be abstractly described. GCStrategy objects must be looked up through +/// the Function. The objects themselves are owned by the Context and must +/// be immutable. +class GCStrategy { +private: + std::string Name; + friend class GCModuleInfo; + +protected: + bool UseStatepoints; /// Uses gc.statepoints as opposed to gc.roots, + /// if set, none of the other options can be + /// anything but their default values. + + unsigned NeededSafePoints; ///< Bitmask of required safe points. + bool CustomReadBarriers; ///< Default is to insert loads. + bool CustomWriteBarriers; ///< Default is to insert stores. + bool CustomRoots; ///< Default is to pass through to backend. + bool InitRoots; ///< If set, roots are nulled during lowering. + bool UsesMetadata; ///< If set, backend must emit metadata tables. + +public: + GCStrategy(); + virtual ~GCStrategy() {} + + /// Return the name of the GC strategy. This is the value of the collector + /// name string specified on functions which use this strategy. + const std::string &getName() const { return Name; } + + /// By default, write barriers are replaced with simple store + /// instructions. If true, you must provide a custom pass to lower + /// calls to @llvm.gcwrite. + bool customWriteBarrier() const { return CustomWriteBarriers; } + + /// By default, read barriers are replaced with simple load + /// instructions. If true, you must provide a custom pass to lower + /// calls to @llvm.gcread. + bool customReadBarrier() const { return CustomReadBarriers; } + + /// Returns true if this strategy is expecting the use of gc.statepoints, + /// and false otherwise. + bool useStatepoints() const { return UseStatepoints; } + + /** @name Statepoint Specific Properties */ + ///@{ + + /// If the value specified can be reliably distinguished, returns true for + /// pointers to GC managed locations and false for pointers to non-GC + /// managed locations. Note a GCStrategy can always return 'None' (i.e. an + /// empty optional indicating it can't reliably distinguish. + virtual Optional<bool> isGCManagedPointer(const Value *V) const { + return None; + } + ///@} + + /** @name GCRoot Specific Properties + * These properties and overrides only apply to collector strategies using + * GCRoot. + */ + ///@{ + + /// True if safe points of any kind are required. By default, none are + /// recorded. + bool needsSafePoints() const { return NeededSafePoints != 0; } + + /// True if the given kind of safe point is required. By default, none are + /// recorded. + bool needsSafePoint(GC::PointKind Kind) const { + return (NeededSafePoints & 1 << Kind) != 0; + } + + /// By default, roots are left for the code generator so it can generate a + /// stack map. If true, you must provide a custom pass to lower + /// calls to @llvm.gcroot. + bool customRoots() const { return CustomRoots; } + + /// If set, gcroot intrinsics should initialize their allocas to null + /// before the first use. This is necessary for most GCs and is enabled by + /// default. + bool initializeRoots() const { return InitRoots; } + + /// If set, appropriate metadata tables must be emitted by the back-end + /// (assembler, JIT, or otherwise). For statepoint, this method is + /// currently unsupported. The stackmap information can be found in the + /// StackMap section as described in the documentation. + bool usesMetadata() const { return UsesMetadata; } + + ///@} +}; + +/// Subclasses of GCStrategy are made available for use during compilation by +/// adding them to the global GCRegistry. This can done either within the +/// LLVM source tree or via a loadable plugin. An example registeration +/// would be: +/// static GCRegistry::Add<CustomGC> X("custom-name", +/// "my custom supper fancy gc strategy"); +/// +/// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also +/// register your GCMetadataPrinter subclass with the +/// GCMetadataPrinterRegistery as well. +typedef Registry<GCStrategy> GCRegistry; } #endif diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h index 51a31842a5c00..5207f801c84ea 100644 --- a/include/llvm/CodeGen/GCs.h +++ b/include/llvm/CodeGen/GCs.h @@ -15,29 +15,32 @@ #define LLVM_CODEGEN_GCS_H namespace llvm { - class GCStrategy; - class GCMetadataPrinter; +class GCStrategy; +class GCMetadataPrinter; - /// FIXME: Collector instances are not useful on their own. These no longer - /// serve any purpose except to link in the plugins. +/// FIXME: Collector instances are not useful on their own. These no longer +/// serve any purpose except to link in the plugins. - /// Creates an ocaml-compatible garbage collector. - void linkOcamlGC(); +/// Creates a CoreCLR-compatible garbage collector. +void linkCoreCLRGC(); - /// Creates an ocaml-compatible metadata printer. - void linkOcamlGCPrinter(); +/// Creates an ocaml-compatible garbage collector. +void linkOcamlGC(); - /// Creates an erlang-compatible garbage collector. - void linkErlangGC(); +/// Creates an ocaml-compatible metadata printer. +void linkOcamlGCPrinter(); - /// Creates an erlang-compatible metadata printer. - void linkErlangGCPrinter(); +/// Creates an erlang-compatible garbage collector. +void linkErlangGC(); - /// Creates a shadow stack garbage collector. This collector requires no code - /// generator support. - void linkShadowStackGC(); +/// Creates an erlang-compatible metadata printer. +void linkErlangGCPrinter(); - void linkStatepointExampleGC(); +/// Creates a shadow stack garbage collector. This collector requires no code +/// generator support. +void linkShadowStackGC(); + +void linkStatepointExampleGC(); } #endif diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 952362ed6ce3b..c2071fe5dd2d7 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -229,11 +229,18 @@ namespace ISD { SMULO, UMULO, /// Simple binary floating point operators. - FADD, FSUB, FMUL, FMA, FDIV, FREM, + FADD, FSUB, FMUL, FDIV, FREM, + + /// FMA - Perform a * b + c with no intermediate rounding step. + FMA, + + /// FMAD - Perform a * b + c, while getting the same result as the + /// separately rounded operations. + FMAD, /// FCOPYSIGN(X, Y) - Return the value of X with the sign of Y. NOTE: This - /// DAG node does not require that X and Y have the same type, just that the - /// are both floating point. X and the result must have the same type. + /// DAG node does not require that X and Y have the same type, just that + /// they are both floating point. X and the result must have the same type. /// FCOPYSIGN(f32, f64) is allowed. FCOPYSIGN, @@ -301,6 +308,10 @@ namespace ISD { /// part. MULHU, MULHS, + /// [US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned + /// integers. + SMIN, SMAX, UMIN, UMAX, + /// Bitwise operators - logical and, logical or, logical xor. AND, OR, XOR, @@ -680,13 +691,29 @@ namespace ISD { ATOMIC_LOAD_UMIN, ATOMIC_LOAD_UMAX, - // Masked load and store + // Masked load and store - consecutive vector load and store operations + // with additional mask operand that prevents memory accesses to the + // masked-off lanes. MLOAD, MSTORE, + // Masked gather and scatter - load and store operations for a vector of + // random addresses with additional mask operand that prevents memory + // accesses to the masked-off lanes. + MGATHER, MSCATTER, + /// This corresponds to the llvm.lifetime.* intrinsics. The first operand /// is the chain and the second operand is the alloca pointer. LIFETIME_START, LIFETIME_END, + /// GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the + /// beginning and end of GC transition sequence, and carry arbitrary + /// information that target might need for lowering. The first operand is + /// a chain, the rest are specified by the target and not touched by the DAG + /// optimizers. GC_TRANSITION_START..GC_TRANSITION_END pairs may not be + /// nested. + GC_TRANSITION_START, + GC_TRANSITION_END, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END @@ -696,7 +723,7 @@ namespace ISD { /// which do not reference a specific memory location should be less than /// this value. Those that do must not be less than this value, and can /// be used with SelectionDAG::getMemIntrinsicNode. - static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+180; + static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+200; //===--------------------------------------------------------------------===// /// MemIndexedMode enum - This enum defines the load / store indexed diff --git a/include/llvm/CodeGen/JumpInstrTables.h b/include/llvm/CodeGen/JumpInstrTables.h deleted file mode 100644 index 005bc1eb2b2de..0000000000000 --- a/include/llvm/CodeGen/JumpInstrTables.h +++ /dev/null @@ -1,105 +0,0 @@ -//===-- JumpInstrTables.h: Jump-Instruction Tables --------------*- C++ -*-===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief An implementation of tables consisting of jump instructions -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_JUMPINSTRTABLES_H -#define LLVM_CODEGEN_JUMPINSTRTABLES_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/Pass.h" -#include "llvm/Target/TargetOptions.h" - -namespace llvm { -class Constant; -class Function; -class FunctionType; -class JumpInstrTableInfo; -class Module; - -/// A class to manage a set of jump tables indexed on function type. It looks at -/// each function in the module to find all the functions that have the -/// jumptable attribute set. For each such function, it creates a new -/// jump-instruction-table function and stores the mapping in the ImmutablePass -/// JumpInstrTableInfo. -/// -/// These special functions get lowered in AsmPrinter to assembly of the form: -/// \verbatim -/// .globl f -/// .type f,@function -/// .align 8,0x90 -/// f: -/// jmp f_orig@PLT -/// \endverbatim -/// -/// Support for an architecture depends on three functions in TargetInstrInfo: -/// getUnconditionalBranch, getTrap, and getJumpInstrTableEntryBound. AsmPrinter -/// uses these to generate the appropriate instructions for the jump statement -/// (an unconditional branch) and for padding to make the table have a size that -/// is a power of two. This padding uses a trap instruction to ensure that calls -/// to this area halt the program. The default implementations of these -/// functions call llvm_unreachable, except for getJumpInstrTableEntryBound, -/// which returns 0 by default. -class JumpInstrTables : public ModulePass { -public: - static char ID; - - JumpInstrTables(); - JumpInstrTables(JumpTable::JumpTableType JTT); - virtual ~JumpInstrTables(); - bool runOnModule(Module &M) override; - const char *getPassName() const override { return "Jump-Instruction Tables"; } - void getAnalysisUsage(AnalysisUsage &AU) const override; - - /// Creates a jump-instruction table function for the Target and adds it to - /// the tables. - Function *insertEntry(Module &M, Function *Target); - - /// Checks to see if there is already a table for the given FunctionType. - bool hasTable(FunctionType *FunTy); - - /// Maps the function into a subset of function types, depending on the - /// jump-instruction table style selected from JumpTableTypes in - /// JumpInstrTables.cpp. The choice of mapping determines the number of - /// jump-instruction tables generated by this pass. E.g., the simplest mapping - /// converts every function type into void f(); so, all functions end up in a - /// single table. - static FunctionType *transformType(JumpTable::JumpTableType JTT, - FunctionType *FunTy); -private: - /// The metadata used while a jump table is being built - struct TableMeta { - /// The number of this table - unsigned TableNum; - - /// The current number of jump entries in the table. - unsigned Count; - }; - - typedef DenseMap<FunctionType *, struct TableMeta> JumpMap; - - /// The current state of functions and jump entries in the table(s). - JumpMap Metadata; - - /// The ImmutablePass that stores information about the generated tables. - JumpInstrTableInfo *JITI; - - /// The total number of tables. - unsigned TableCount; - - /// The type of tables to build. - JumpTable::JumpTableType JTType; -}; - -/// Creates a JumpInstrTables pass for the given type of jump table. -ModulePass *createJumpInstrTablesPass(JumpTable::JumpTableType JTT); -} - -#endif /* LLVM_CODEGEN_JUMPINSTRTABLES_H */ diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h index cf601ae5384c3..f347f66e0981c 100644 --- a/include/llvm/CodeGen/LatencyPriorityQueue.h +++ b/include/llvm/CodeGen/LatencyPriorityQueue.h @@ -83,8 +83,6 @@ namespace llvm { void remove(SUnit *SU) override; - void dump(ScheduleDAG* DAG) const override; - // scheduledNode - As nodes are scheduled, we look to see if there are any // successor nodes that have a single unscheduled predecessor. If so, that // single predecessor has a higher priority, since scheduling it will make diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h index 11a360a491a73..7d7e48af2a0ff 100644 --- a/include/llvm/CodeGen/LexicalScopes.h +++ b/include/llvm/CodeGen/LexicalScopes.h @@ -23,7 +23,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/DebugLoc.h" -#include "llvm/IR/Metadata.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/ValueHandle.h" #include <unordered_map> #include <utility> @@ -45,7 +45,8 @@ typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange; class LexicalScope { public: - LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A) + LexicalScope(LexicalScope *P, const DILocalScope *D, const DILocation *I, + bool A) : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A), LastInsn(nullptr), FirstInsn(nullptr), DFSIn(0), DFSOut(0) { assert((!D || D->isResolved()) && "Expected resolved node"); @@ -57,8 +58,8 @@ public: // Accessors. LexicalScope *getParent() const { return Parent; } const MDNode *getDesc() const { return Desc; } - const MDNode *getInlinedAt() const { return InlinedAtLocation; } - const MDNode *getScopeNode() const { return Desc; } + const DILocation *getInlinedAt() const { return InlinedAtLocation; } + const DILocalScope *getScopeNode() const { return Desc; } bool isAbstractScope() const { return AbstractScope; } SmallVectorImpl<LexicalScope *> &getChildren() { return Children; } SmallVectorImpl<InsnRange> &getRanges() { return Ranges; } @@ -118,8 +119,8 @@ public: private: LexicalScope *Parent; // Parent to this scope. - const MDNode *Desc; // Debug info descriptor. - const MDNode *InlinedAtLocation; // Location at which this + const DILocalScope *Desc; // Debug info descriptor. + const DILocation *InlinedAtLocation; // Location at which this // scope is inlined. bool AbstractScope; // Abstract Scope SmallVector<LexicalScope *, 4> Children; // Scopes defined in scope. @@ -158,16 +159,16 @@ public: /// getMachineBasicBlocks - Populate given set using machine basic blocks /// which have machine instructions that belong to lexical scope identified by /// DebugLoc. - void getMachineBasicBlocks(DebugLoc DL, + void getMachineBasicBlocks(const DILocation *DL, SmallPtrSetImpl<const MachineBasicBlock *> &MBBs); /// dominates - Return true if DebugLoc's lexical scope dominates at least one /// machine instruction's lexical scope in a given machine basic block. - bool dominates(DebugLoc DL, MachineBasicBlock *MBB); + bool dominates(const DILocation *DL, MachineBasicBlock *MBB); /// findLexicalScope - Find lexical scope, either regular or inlined, for the /// given DebugLoc. Return NULL if not found. - LexicalScope *findLexicalScope(DebugLoc DL); + LexicalScope *findLexicalScope(const DILocation *DL); /// getAbstractScopesList - Return a reference to list of abstract scopes. ArrayRef<LexicalScope *> getAbstractScopesList() const { @@ -175,17 +176,19 @@ public: } /// findAbstractScope - Find an abstract scope or return null. - LexicalScope *findAbstractScope(const MDNode *N) { + LexicalScope *findAbstractScope(const DILocalScope *N) { auto I = AbstractScopeMap.find(N); return I != AbstractScopeMap.end() ? &I->second : nullptr; } - /// findInlinedScope - Find an inlined scope for the given DebugLoc or return - /// NULL. - LexicalScope *findInlinedScope(DebugLoc DL); + /// findInlinedScope - Find an inlined scope for the given scope/inlined-at. + LexicalScope *findInlinedScope(const DILocalScope *N, const DILocation *IA) { + auto I = InlinedLexicalScopeMap.find(std::make_pair(N, IA)); + return I != InlinedLexicalScopeMap.end() ? &I->second : nullptr; + } /// findLexicalScope - Find regular lexical scope or return null. - LexicalScope *findLexicalScope(const MDNode *N) { + LexicalScope *findLexicalScope(const DILocalScope *N) { auto I = LexicalScopeMap.find(N); return I != LexicalScopeMap.end() ? &I->second : nullptr; } @@ -194,18 +197,24 @@ public: void dump(); /// getOrCreateAbstractScope - Find or create an abstract lexical scope. - LexicalScope *getOrCreateAbstractScope(const MDNode *N); + LexicalScope *getOrCreateAbstractScope(const DILocalScope *Scope); private: - /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If + /// getOrCreateLexicalScope - Find lexical scope for the given Scope/IA. If /// not available then create new lexical scope. - LexicalScope *getOrCreateLexicalScope(DebugLoc DL); + LexicalScope *getOrCreateLexicalScope(const DILocalScope *Scope, + const DILocation *IA = nullptr); + LexicalScope *getOrCreateLexicalScope(const DILocation *DL) { + return DL ? getOrCreateLexicalScope(DL->getScope(), DL->getInlinedAt()) + : nullptr; + } /// getOrCreateRegularScope - Find or create a regular lexical scope. - LexicalScope *getOrCreateRegularScope(MDNode *Scope); + LexicalScope *getOrCreateRegularScope(const DILocalScope *Scope); /// getOrCreateInlinedScope - Find or create an inlined lexical scope. - LexicalScope *getOrCreateInlinedScope(MDNode *Scope, MDNode *InlinedAt); + LexicalScope *getOrCreateInlinedScope(const DILocalScope *Scope, + const DILocation *InlinedAt); /// extractLexicalScopes - Extract instruction ranges for each lexical scopes /// for the given machine function. @@ -221,17 +230,18 @@ private: /// LexicalScopeMap - Tracks the scopes in the current function. // Use an unordered_map to ensure value pointer validity over insertion. - std::unordered_map<const MDNode *, LexicalScope> LexicalScopeMap; + std::unordered_map<const DILocalScope *, LexicalScope> LexicalScopeMap; /// InlinedLexicalScopeMap - Tracks inlined function scopes in current /// function. - std::unordered_map<std::pair<const MDNode *, const MDNode *>, LexicalScope, - pair_hash<const MDNode *, const MDNode *>> - InlinedLexicalScopeMap; + std::unordered_map<std::pair<const DILocalScope *, const DILocation *>, + LexicalScope, + pair_hash<const DILocalScope *, const DILocation *>> + InlinedLexicalScopeMap; /// AbstractScopeMap - These scopes are not included LexicalScopeMap. // Use an unordered_map to ensure value pointer validity over insertion. - std::unordered_map<const MDNode *, LexicalScope> AbstractScopeMap; + std::unordered_map<const DILocalScope *, LexicalScope> AbstractScopeMap; /// AbstractScopesList - Tracks abstract scopes constructed while processing /// a function. diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h index e7ccbfa617e52..fee131e4a3c6b 100644 --- a/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -36,6 +36,7 @@ namespace { (void) llvm::createGreedyRegisterAllocator(); (void) llvm::createDefaultPBQPRegisterAllocator(); + llvm::linkCoreCLRGC(); llvm::linkOcamlGC(); llvm::linkErlangGC(); llvm::linkShadowStackGC(); diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index ce9845ee1673a..9b8b91c9b80e2 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -27,6 +27,7 @@ #include "llvm/Support/Allocator.h" #include <cassert> #include <climits> +#include <set> namespace llvm { class CoalescerPair; @@ -194,6 +195,12 @@ namespace llvm { Segments segments; // the liveness segments VNInfoList valnos; // value#'s + // The segment set is used temporarily to accelerate initial computation + // of live ranges of physical registers in computeRegUnitRange. + // After that the set is flushed to the segment vector and deleted. + typedef std::set<Segment> SegmentSet; + std::unique_ptr<SegmentSet> segmentSet; + typedef Segments::iterator iterator; iterator begin() { return segments.begin(); } iterator end() { return segments.end(); } @@ -211,12 +218,16 @@ namespace llvm { const_vni_iterator vni_end() const { return valnos.end(); } /// Constructs a new LiveRange object. - LiveRange() { - } + LiveRange(bool UseSegmentSet = false) + : segmentSet(UseSegmentSet ? llvm::make_unique<SegmentSet>() + : nullptr) {} /// Constructs a new LiveRange object by copying segments and valnos from /// another LiveRange. LiveRange(const LiveRange &Other, BumpPtrAllocator &Allocator) { + assert(Other.segmentSet == nullptr && + "Copying of LiveRanges with active SegmentSets is not supported"); + // Duplicate valnos. for (const VNInfo *VNI : Other.valnos) { createValueCopy(VNI, Allocator); @@ -437,14 +448,12 @@ namespace llvm { /// Add the specified Segment to this range, merging segments as /// appropriate. This returns an iterator to the inserted segment (which /// may have grown since it was inserted). - iterator addSegment(Segment S) { - return addSegmentFrom(S, segments.begin()); - } + iterator addSegment(Segment S); - /// extendInBlock - If this range is live before Kill in the basic block - /// that starts at StartIdx, extend it to be live up to Kill, and return - /// the value. If there is no segment before Kill, return NULL. - VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Kill); + /// If this range is live before @p Use in the basic block that starts at + /// @p StartIdx, extend it to be live up to @p Use, and return the value. If + /// there is no segment before @p Use, return nullptr. + VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Use); /// join - Join two live ranges (this, and other) together. This applies /// mappings to the value numbers in the LHS/RHS ranges as specified. If @@ -540,6 +549,12 @@ namespace llvm { return thisIndex < otherIndex; } + /// Flush segment set into the regular segment vector. + /// The method is to be called after the live range + /// has been created, if use of the segment set was + /// activated in the constructor of the live range. + void flushSegmentSet(); + void print(raw_ostream &OS) const; void dump() const; @@ -557,10 +572,8 @@ namespace llvm { void append(const LiveRange::Segment S); private: - - iterator addSegmentFrom(Segment S, iterator From); - void extendSegmentEndTo(iterator I, SlotIndex NewEnd); - iterator extendSegmentStartTo(iterator I, SlotIndex NewStr); + friend class LiveRangeUpdater; + void addSegmentToSet(Segment S); void markValNoForDeletion(VNInfo *V); }; @@ -606,6 +619,10 @@ namespace llvm { LiveInterval(unsigned Reg, float Weight) : SubRanges(nullptr), reg(Reg), weight(Weight) {} + ~LiveInterval() { + clearSubRanges(); + } + template<typename T> class SingleLinkedListIterator { T *P; @@ -681,9 +698,7 @@ namespace llvm { } /// Removes all subregister liveness information. - void clearSubRanges() { - SubRanges = nullptr; - } + void clearSubRanges(); /// Removes all subranges without any segments (subranges without segments /// are not considered valid and should only exist temporarily). @@ -726,13 +741,14 @@ namespace llvm { #endif private: - LiveInterval& operator=(const LiveInterval& rhs) LLVM_DELETED_FUNCTION; - /// Appends @p Range to SubRanges list. void appendSubRange(SubRange *Range) { Range->Next = SubRanges; SubRanges = Range; } + + /// Free memory held by SubRange. + void freeSubRange(SubRange *S); }; inline raw_ostream &operator<<(raw_ostream &OS, const LiveInterval &LI) { diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h index d8c921fce3137..9673f80e08566 100644 --- a/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h @@ -27,12 +27,15 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Target/TargetRegisterInfo.h" #include <cmath> #include <iterator> namespace llvm { +extern cl::opt<bool> UseSegmentSetForPhysRegs; + class AliasAnalysis; class BitVector; class BlockFrequency; @@ -97,7 +100,7 @@ namespace llvm { public: static char ID; // Pass identification, replacement for typeid LiveIntervals(); - virtual ~LiveIntervals(); + ~LiveIntervals() override; // Calculate the spill weight to assign to a single instruction. static float getSpillWeight(bool isDef, bool isUse, @@ -181,12 +184,6 @@ namespace llvm { void pruneValue(LiveRange &LR, SlotIndex Kill, SmallVectorImpl<SlotIndex> *EndPoints); - /// Subregister aware variant of pruneValue(LiveRange &LR, SlotIndex Kill, - /// SmallVectorImpl<SlotIndex> &EndPoints). Prunes the value in the main - /// range and all sub ranges. - void pruneValue(LiveInterval &LI, SlotIndex Kill, - SmallVectorImpl<SlotIndex> *EndPoints); - SlotIndexes *getSlotIndexes() const { return Indexes; } @@ -383,7 +380,8 @@ namespace llvm { LiveRange *LR = RegUnitRanges[Unit]; if (!LR) { // Compute missing ranges on demand. - RegUnitRanges[Unit] = LR = new LiveRange(); + // Use segment set to speed-up initial computation of the live range. + RegUnitRanges[Unit] = LR = new LiveRange(UseSegmentSetForPhysRegs); computeRegUnitRange(*LR, Unit); } return *LR; @@ -399,6 +397,15 @@ namespace llvm { return RegUnitRanges[Unit]; } + /// Remove value numbers and related live segments starting at position + /// @p Pos that are part of any liverange of physical register @p Reg or one + /// of its subregisters. + void removePhysRegDefAt(unsigned Reg, SlotIndex Pos); + + /// Remove value number and related live segments of @p LI and its subranges + /// that start at position @p Pos. + void removeVRegDefAt(LiveInterval &LI, SlotIndex Pos); + private: /// Compute live intervals for all virtual registers. void computeVirtRegs(); diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h index 1381c46a2750e..967f0cbac7194 100644 --- a/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/include/llvm/CodeGen/LiveIntervalUnion.h @@ -179,8 +179,8 @@ public: } private: - Query(const Query&) LLVM_DELETED_FUNCTION; - void operator=(const Query&) LLVM_DELETED_FUNCTION; + Query(const Query&) = delete; + void operator=(const Query&) = delete; }; // Array of LiveIntervalUnions. diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h index 91e4ddcde1706..6ffd3eeea351f 100644 --- a/include/llvm/CodeGen/LivePhysRegs.h +++ b/include/llvm/CodeGen/LivePhysRegs.h @@ -44,8 +44,8 @@ class LivePhysRegs { const TargetRegisterInfo *TRI; SparseSet<unsigned> LiveRegs; - LivePhysRegs(const LivePhysRegs&) LLVM_DELETED_FUNCTION; - LivePhysRegs &operator=(const LivePhysRegs&) LLVM_DELETED_FUNCTION; + LivePhysRegs(const LivePhysRegs&) = delete; + LivePhysRegs &operator=(const LivePhysRegs&) = delete; public: /// \brief Constructs a new empty LivePhysRegs set. LivePhysRegs() : TRI(nullptr), LiveRegs() {} @@ -57,9 +57,9 @@ public: } /// \brief Clear and initialize the LivePhysRegs set. - void init(const TargetRegisterInfo *_TRI) { - assert(_TRI && "Invalid TargetRegisterInfo pointer."); - TRI = _TRI; + void init(const TargetRegisterInfo *TRI) { + assert(TRI && "Invalid TargetRegisterInfo pointer."); + this->TRI = TRI; LiveRegs.clear(); LiveRegs.setUniverse(TRI->getNumRegs()); } @@ -93,7 +93,8 @@ public: } /// \brief Removes physical registers clobbered by the regmask operand @p MO. - void removeRegsInMask(const MachineOperand &MO); + void removeRegsInMask(const MachineOperand &MO, + SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> *Clobbers); /// \brief Returns true if register @p Reg is contained in the set. This also /// works if only the super register of @p Reg has been defined, because we @@ -109,7 +110,11 @@ public: /// instruction(bundle): Remove killed-uses, add defs. This is the not /// recommended way, because it depends on accurate kill flags. If possible /// use stepBackwards() instead of this function. - void stepForward(const MachineInstr &MI); + /// The clobbers set will be the list of registers either defined or clobbered + /// by a regmask. The operand will identify whether this is a regmask or + /// register operand. + void stepForward(const MachineInstr &MI, + SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> &Clobbers); /// \brief Adds all live-in registers of basic block @p MBB. void addLiveIns(const MachineBasicBlock *MBB) { diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h index 44c3c4eaf7b17..de855f2fe7a0f 100644 --- a/include/llvm/CodeGen/LiveRangeEdit.h +++ b/include/llvm/CodeGen/LiveRangeEdit.h @@ -122,7 +122,7 @@ public: MRI.setDelegate(this); } - ~LiveRangeEdit() { MRI.resetDelegate(this); } + ~LiveRangeEdit() override { MRI.resetDelegate(this); } LiveInterval &getParent() const { assert(Parent && "No parent LiveInterval"); diff --git a/include/llvm/CodeGen/LiveStackAnalysis.h b/include/llvm/CodeGen/LiveStackAnalysis.h index df683983fa53e..f495507c66ecd 100644 --- a/include/llvm/CodeGen/LiveStackAnalysis.h +++ b/include/llvm/CodeGen/LiveStackAnalysis.h @@ -21,6 +21,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Target/TargetRegisterInfo.h" #include <map> +#include <unordered_map> namespace llvm { @@ -33,7 +34,7 @@ namespace llvm { /// S2IMap - Stack slot indices to live interval mapping. /// - typedef std::map<int, LiveInterval> SS2IntervalMap; + typedef std::unordered_map<int, LiveInterval> SS2IntervalMap; SS2IntervalMap S2IMap; /// S2RCMap - Stack slot indices to register class mapping. diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index 1440b967aea21..357aef0ee60da 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -315,9 +315,18 @@ public: // LiveIn management methods. - /// addLiveIn - Add the specified register as a live in. Note that it - /// is an error to add the same register to the same set more than once. - void addLiveIn(unsigned Reg) { LiveIns.push_back(Reg); } + /// Adds the specified register as a live in. Note that it is an error to add + /// the same register to the same set more than once unless the intention is + /// to call sortUniqueLiveIns after all registers are added. + void addLiveIn(unsigned Reg) { LiveIns.push_back(Reg); } + + /// Sorts and uniques the LiveIns vector. It can be significantly faster to do + /// this than repeatedly calling isLiveIn before calling addLiveIn for every + /// LiveIn insertion. + void sortUniqueLiveIns() { + std::sort(LiveIns.begin(), LiveIns.end()); + LiveIns.erase(std::unique(LiveIns.begin(), LiveIns.end()), LiveIns.end()); + } /// Add PhysReg as live in to this block, and ensure that there is a copy of /// PhysReg to a virtual register of class RC. Return the virtual register @@ -624,17 +633,18 @@ public: ///< neighborhood. }; - /// computeRegisterLiveness - Return whether (physical) register \c Reg - /// has been <def>ined and not <kill>ed as of just before \c MI. + /// Return whether (physical) register \p Reg has been <def>ined and not + /// <kill>ed as of just before \p Before. /// - /// Search is localised to a neighborhood of - /// \c Neighborhood instructions before (searching for defs or kills) and - /// Neighborhood instructions after (searching just for defs) MI. + /// Search is localised to a neighborhood of \p Neighborhood instructions + /// before (searching for defs or kills) and \p Neighborhood instructions + /// after (searching just for defs) \p Before. /// - /// \c Reg must be a physical register. + /// \p Reg must be a physical register. LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, - unsigned Reg, MachineInstr *MI, - unsigned Neighborhood=10); + unsigned Reg, + const_iterator Before, + unsigned Neighborhood=10) const; // Debugging methods. void dump() const; diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index 1aef689eb7a44..feb394e7a69e0 100644 --- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -35,7 +35,7 @@ public: MachineBlockFrequencyInfo(); - ~MachineBlockFrequencyInfo(); + ~MachineBlockFrequencyInfo() override; void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/include/llvm/CodeGen/MachineDominanceFrontier.h b/include/llvm/CodeGen/MachineDominanceFrontier.h index e099e716c63da..4131194a0c0fc 100644 --- a/include/llvm/CodeGen/MachineDominanceFrontier.h +++ b/include/llvm/CodeGen/MachineDominanceFrontier.h @@ -26,8 +26,8 @@ public: typedef DominanceFrontierBase<MachineBasicBlock>::iterator iterator; typedef DominanceFrontierBase<MachineBasicBlock>::const_iterator const_iterator; - void operator=(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION; - MachineDominanceFrontier(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION; + void operator=(const MachineDominanceFrontier &) = delete; + MachineDominanceFrontier(const MachineDominanceFrontier &) = delete; static char ID; diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h index a6980a6daeacd..4428fa618fb01 100644 --- a/include/llvm/CodeGen/MachineDominators.h +++ b/include/llvm/CodeGen/MachineDominators.h @@ -45,9 +45,6 @@ class MachineDominatorTree : public MachineFunctionPass { MachineBasicBlock *FromBB; MachineBasicBlock *ToBB; MachineBasicBlock *NewBB; - CriticalEdge(MachineBasicBlock *FromBB, MachineBasicBlock *ToBB, - MachineBasicBlock *NewBB) - : FromBB(FromBB), ToBB(ToBB), NewBB(NewBB) {} }; /// \brief Pile up all the critical edges to be split. @@ -67,74 +64,7 @@ class MachineDominatorTree : public MachineFunctionPass { /// the fast query path of DT as much as possible. /// /// \post CriticalEdgesToSplit.empty(). - void applySplitCriticalEdges() const { - // Bail out early if there is nothing to do. - if (CriticalEdgesToSplit.empty()) - return; - - // For each element in CriticalEdgesToSplit, remember whether or - // not element is the new immediate domminator of its successor. - // The mapping is done by index, i.e., the information for the ith - // element of CriticalEdgesToSplit is the ith element of IsNewIDom. - SmallVector<bool, 32> IsNewIDom; - IsNewIDom.resize(CriticalEdgesToSplit.size()); - size_t Idx = 0; - - // Collect all the dominance properties info, before invalidating - // the underlying DT. - for (CriticalEdge &Edge : CriticalEdgesToSplit) { - // Update dominator information. - MachineBasicBlock *Succ = Edge.ToBB; - MachineDomTreeNode *SucccDTNode = DT->getNode(Succ); - - IsNewIDom[Idx] = true; - for (MachineBasicBlock *PredBB : Succ->predecessors()) { - if (PredBB == Edge.NewBB) - continue; - // If we are in this situation: - // FromBB1 FromBB2 - // + + - // + + + + - // + + + + - // ... Split1 Split2 ... - // + + - // + + - // + - // Succ - // Instead of checking the domiance property with Split2, we - // check it with FromBB2 since Split2 is still unknown of the - // underlying DT structure. - if (NewBBs.count(PredBB)) { - assert(PredBB->pred_size() == 1 && "A basic block resulting from a " - "critical edge split has more " - "than one predecessor!"); - PredBB = *PredBB->pred_begin(); - } - if (!DT->dominates(SucccDTNode, DT->getNode(PredBB))) { - IsNewIDom[Idx] = false; - break; - } - } - ++Idx; - } - - // Now, update DT with the collected dominance properties info. - Idx = 0; - for (CriticalEdge &Edge : CriticalEdgesToSplit) { - // We know FromBB dominates NewBB. - MachineDomTreeNode *NewDTNode = DT->addNewBlock(Edge.NewBB, Edge.FromBB); - MachineDomTreeNode *SucccDTNode = DT->getNode(Edge.ToBB); - - // If all the other predecessors of "Succ" are dominated by "Succ" itself - // then the new block is the new immediate dominator of "Succ". Otherwise, - // the new block doesn't dominate anything. - if (IsNewIDom[Idx]) - DT->changeImmediateDominator(SucccDTNode, NewDTNode); - ++Idx; - } - NewBBs.clear(); - CriticalEdgesToSplit.clear(); - } + void applySplitCriticalEdges() const; public: static char ID; // Pass ID, replacement for typeid @@ -142,7 +72,7 @@ public: MachineDominatorTree(); - ~MachineDominatorTree(); + ~MachineDominatorTree() override; DominatorTreeBase<MachineBasicBlock> &getBase() { applySplitCriticalEdges(); @@ -307,7 +237,7 @@ public: (void)Inserted; assert(Inserted && "A basic block inserted via edge splitting cannot appear twice"); - CriticalEdgesToSplit.push_back(CriticalEdge(FromBB, ToBB, NewBB)); + CriticalEdgesToSplit.push_back({FromBB, ToBB, NewBB}); } }; diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index 667736021f927..40f3b4944cc1e 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -246,6 +246,21 @@ class MachineFrameInfo { /// True if this is a varargs function that contains a musttail call. bool HasMustTailInVarArgFunc; + /// True if this function contains a tail call. If so immutable objects like + /// function arguments are no longer so. A tail call *can* override fixed + /// stack objects like arguments so we can't treat them as immutable. + bool HasTailCall; + + /// Not null, if shrink-wrapping found a better place for the prologue. + MachineBasicBlock *Save; + /// Not null, if shrink-wrapping found a better place for the epilogue. + MachineBasicBlock *Restore; + + /// Check if it exists a path from \p MBB leading to the basic + /// block with a SavePoint (a.k.a. prologue). + bool isBeforeSavePoint(const MachineFunction &MF, + const MachineBasicBlock &MBB) const; + public: explicit MachineFrameInfo(unsigned StackAlign, bool isStackRealign, bool RealignOpt) @@ -269,6 +284,9 @@ public: HasInlineAsmWithSPAdjust = false; HasVAStart = false; HasMustTailInVarArgFunc = false; + Save = nullptr; + Restore = nullptr; + HasTailCall = false; } /// hasStackObjects - Return true if there are any stack objects in this @@ -496,6 +514,10 @@ public: bool hasMustTailInVarArgFunc() const { return HasMustTailInVarArgFunc; } void setHasMustTailInVarArgFunc(bool B) { HasMustTailInVarArgFunc = B; } + /// Returns true if the function contains a tail call. + bool hasTailCall() const { return HasTailCall; } + void setHasTailCall() { HasTailCall = true; } + /// getMaxCallFrameSize - Return the maximum size of a call frame that must be /// allocated for an outgoing function call. This is only available if /// CallFrameSetup/Destroy pseudo instructions are used by the target, and @@ -516,10 +538,6 @@ public: /// on the stack. Returns an index with a negative value. int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset); - /// Allocates memory at a fixed, target-specific offset from the frame - /// pointer. Marks the function as having its frame address taken. - int CreateFrameAllocation(uint64_t Size); - /// isFixedObjectIndex - Returns true if the specified index corresponds to a /// fixed stack object. bool isFixedObjectIndex(int ObjectIdx) const { @@ -537,6 +555,9 @@ public: /// isImmutableObjectIndex - Returns true if the specified index corresponds /// to an immutable object. bool isImmutableObjectIndex(int ObjectIdx) const { + // Tail calling functions can clobber their function arguments. + if (HasTailCall) + return false; assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); return Objects[ObjectIdx+NumFixedObjects].isImmutable; @@ -601,6 +622,11 @@ public: void setCalleeSavedInfoValid(bool v) { CSIValid = v; } + MachineBasicBlock *getSavePoint() const { return Save; } + void setSavePoint(MachineBasicBlock *NewSave) { Save = NewSave; } + MachineBasicBlock *getRestorePoint() const { return Restore; } + void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; } + /// getPristineRegs - Return a set of physical registers that are pristine on /// entry to the MBB. /// diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index 4e9ff9ebb4fea..94610cabf566d 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -145,8 +145,8 @@ class MachineFunction { /// True if the function includes any inline assembly. bool HasInlineAsm; - MachineFunction(const MachineFunction &) LLVM_DELETED_FUNCTION; - void operator=(const MachineFunction&) LLVM_DELETED_FUNCTION; + MachineFunction(const MachineFunction &) = delete; + void operator=(const MachineFunction&) = delete; public: MachineFunction(const Function *Fn, const TargetMachine &TM, unsigned FunctionNum, MachineModuleInfo &MMI); @@ -176,6 +176,13 @@ public: const TargetSubtargetInfo &getSubtarget() const { return *STI; } void setSubtarget(const TargetSubtargetInfo *ST) { STI = ST; } + /// getSubtarget - This method returns a pointer to the specified type of + /// TargetSubtargetInfo. In debug builds, it verifies that the object being + /// returned is of the correct type. + template<typename STC> const STC &getSubtarget() const { + return *static_cast<const STC *>(STI); + } + /// getRegInfo - Return information about the registers currently in use. /// MachineRegisterInfo &getRegInfo() { return *RegInfo; } diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h index 36f1c6627096c..023eeb1b4d0ea 100644 --- a/include/llvm/CodeGen/MachineFunctionAnalysis.h +++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h @@ -31,7 +31,7 @@ private: public: static char ID; explicit MachineFunctionAnalysis(const TargetMachine &tm); - ~MachineFunctionAnalysis(); + ~MachineFunctionAnalysis() override; MachineFunction &getMF() const { return *MF; } diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index bcf1f5caaa8c6..e57257c76bc70 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -42,7 +42,7 @@ class MachineFunction; class MachineMemOperand; //===----------------------------------------------------------------------===// -/// MachineInstr - Representation of each machine instruction. +/// Representation of each machine instruction. /// /// This class isn't a POD type, but it must have a trivial destructor. When a /// MachineFunction is deleted, all the contained MachineInstrs are deallocated @@ -93,25 +93,25 @@ private: DebugLoc debugLoc; // Source line information. - MachineInstr(const MachineInstr&) LLVM_DELETED_FUNCTION; - void operator=(const MachineInstr&) LLVM_DELETED_FUNCTION; + MachineInstr(const MachineInstr&) = delete; + void operator=(const MachineInstr&) = delete; // Use MachineFunction::DeleteMachineInstr() instead. - ~MachineInstr() LLVM_DELETED_FUNCTION; + ~MachineInstr() = delete; // Intrusive list support friend struct ilist_traits<MachineInstr>; friend struct ilist_traits<MachineBasicBlock>; void setParent(MachineBasicBlock *P) { Parent = P; } - /// MachineInstr ctor - This constructor creates a copy of the given + /// This constructor creates a copy of the given /// MachineInstr in the given MachineFunction. MachineInstr(MachineFunction &, const MachineInstr &); - /// MachineInstr ctor - This constructor create a MachineInstr and add the - /// implicit operands. It reserves space for number of operands specified by + /// This constructor create a MachineInstr and add the implicit operands. + /// It reserves space for number of operands specified by /// MCInstrDesc. An explicit DebugLoc is supplied. - MachineInstr(MachineFunction&, const MCInstrDesc &MCID, - const DebugLoc dl, bool NoImp = false); + MachineInstr(MachineFunction &, const MCInstrDesc &MCID, DebugLoc dl, + bool NoImp = false); // MachineInstrs are pool-allocated and owned by MachineFunction. friend class MachineFunction; @@ -120,43 +120,38 @@ public: const MachineBasicBlock* getParent() const { return Parent; } MachineBasicBlock* getParent() { return Parent; } - /// getAsmPrinterFlags - Return the asm printer flags bitvector. - /// + /// Return the asm printer flags bitvector. uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } - /// clearAsmPrinterFlags - clear the AsmPrinter bitvector - /// + /// Clear the AsmPrinter bitvector. void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } - /// getAsmPrinterFlag - Return whether an AsmPrinter flag is set. - /// + /// Return whether an AsmPrinter flag is set. bool getAsmPrinterFlag(CommentFlag Flag) const { return AsmPrinterFlags & Flag; } - /// setAsmPrinterFlag - Set a flag for the AsmPrinter. - /// + /// Set a flag for the AsmPrinter. void setAsmPrinterFlag(CommentFlag Flag) { AsmPrinterFlags |= (uint8_t)Flag; } - /// clearAsmPrinterFlag - clear specific AsmPrinter flags - /// + /// Clear specific AsmPrinter flags. void clearAsmPrinterFlag(CommentFlag Flag) { AsmPrinterFlags &= ~Flag; } - /// getFlags - Return the MI flags bitvector. + /// Return the MI flags bitvector. uint8_t getFlags() const { return Flags; } - /// getFlag - Return whether an MI flag is set. + /// Return whether an MI flag is set. bool getFlag(MIFlag Flag) const { return Flags & Flag; } - /// setFlag - Set a MI flag. + /// Set a MI flag. void setFlag(MIFlag Flag) { Flags |= (uint8_t)Flag; } @@ -172,8 +167,7 @@ public: Flags &= ~((uint8_t)Flag); } - /// isInsideBundle - Return true if MI is in a bundle (but not the first MI - /// in a bundle). + /// Return true if MI is in a bundle (but not the first MI in a bundle). /// /// A bundle looks like this before it's finalized: /// ---------------- @@ -212,7 +206,7 @@ public: return getFlag(BundledPred); } - /// isBundled - Return true if this instruction part of a bundle. This is true + /// Return true if this instruction part of a bundle. This is true /// if either itself or its following instruction is marked "InsideBundle". bool isBundled() const { return isBundledWithPred() || isBundledWithSucc(); @@ -240,30 +234,25 @@ public: /// Break bundle below this instruction. void unbundleFromSucc(); - /// getDebugLoc - Returns the debug location id of this MachineInstr. - /// - DebugLoc getDebugLoc() const { return debugLoc; } + /// Returns the debug location id of this MachineInstr. + const DebugLoc &getDebugLoc() const { return debugLoc; } - /// \brief Return the debug variable referenced by + /// Return the debug variable referenced by /// this DBG_VALUE instruction. - DIVariable getDebugVariable() const { + const DILocalVariable *getDebugVariable() const { assert(isDebugValue() && "not a DBG_VALUE"); - DIVariable Var(getOperand(2).getMetadata()); - assert(Var.Verify() && "not a DIVariable"); - return Var; + return cast<DILocalVariable>(getOperand(2).getMetadata()); } - /// \brief Return the complex address expression referenced by + /// Return the complex address expression referenced by /// this DBG_VALUE instruction. - DIExpression getDebugExpression() const { + const DIExpression *getDebugExpression() const { assert(isDebugValue() && "not a DBG_VALUE"); - DIExpression Expr(getOperand(3).getMetadata()); - assert(Expr.Verify() && "not a DIExpression"); - return Expr; + return cast<DIExpression>(getOperand(3).getMetadata()); } - /// emitError - Emit an error referring to the source location of this - /// instruction. This should only be used for inline assembly that is somehow + /// Emit an error referring to the source location of this instruction. + /// This should only be used for inline assembly that is somehow /// impossible to compile. Other errors should have been handled much /// earlier. /// @@ -271,13 +260,11 @@ public: /// void emitError(StringRef Msg) const; - /// getDesc - Returns the target instruction descriptor of this - /// MachineInstr. + /// Returns the target instruction descriptor of this MachineInstr. const MCInstrDesc &getDesc() const { return *MCID; } - /// getOpcode - Returns the opcode of this MachineInstr. - /// - int getOpcode() const { return MCID->Opcode; } + /// Returns the opcode of this MachineInstr. + unsigned getOpcode() const { return MCID->Opcode; } /// Access to explicit operands of the instruction. /// @@ -292,8 +279,7 @@ public: return Operands[i]; } - /// getNumExplicitOperands - Returns the number of non-implicit operands. - /// + /// Returns the number of non-implicit operands. unsigned getNumExplicitOperands() const; /// iterator/begin/end - Iterate over all operands of a machine instruction. @@ -357,8 +343,7 @@ public: return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end()); } - /// hasOneMemOperand - Return true if this instruction has exactly one - /// MachineMemOperand. + /// Return true if this instruction has exactly one MachineMemOperand. bool hasOneMemOperand() const { return NumMemRefs == 1; } @@ -372,7 +357,7 @@ public: AllInBundle // Return true if all instructions in bundle have property }; - /// hasProperty - Return true if the instruction (or in the case of a bundle, + /// Return true if the instruction (or in the case of a bundle, /// the instructions inside the bundle) has the specified property. /// The first argument is the property being queried. /// The second argument indicates whether the query should look inside @@ -386,23 +371,22 @@ public: return hasPropertyInBundle(1 << MCFlag, Type); } - /// isVariadic - Return true if this instruction can have a variable number of - /// operands. In this case, the variable operands will be after the normal + /// Return true if this instruction can have a variable number of operands. + /// In this case, the variable operands will be after the normal /// operands but before the implicit definitions and uses (if any are /// present). bool isVariadic(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::Variadic, Type); } - /// hasOptionalDef - Set if this instruction has an optional definition, e.g. + /// Set if this instruction has an optional definition, e.g. /// ARM instructions which can set condition code if 's' bit is set. bool hasOptionalDef(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::HasOptionalDef, Type); } - /// isPseudo - Return true if this is a pseudo instruction that doesn't + /// Return true if this is a pseudo instruction that doesn't /// correspond to a real machine instruction. - /// bool isPseudo(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::Pseudo, Type); } @@ -415,16 +399,15 @@ public: return hasProperty(MCID::Call, Type); } - /// isBarrier - Returns true if the specified instruction stops control flow + /// Returns true if the specified instruction stops control flow /// from executing the instruction immediately following it. Examples include /// unconditional branches and return instructions. bool isBarrier(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Barrier, Type); } - /// isTerminator - Returns true if this instruction part of the terminator for - /// a basic block. Typically this is things like return and branch - /// instructions. + /// Returns true if this instruction part of the terminator for a basic block. + /// Typically this is things like return and branch instructions. /// /// Various passes use this to insert code into the bottom of a basic block, /// but before control flow occurs. @@ -432,21 +415,21 @@ public: return hasProperty(MCID::Terminator, Type); } - /// isBranch - Returns true if this is a conditional, unconditional, or - /// indirect branch. Predicates below can be used to discriminate between + /// Returns true if this is a conditional, unconditional, or indirect branch. + /// Predicates below can be used to discriminate between /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to /// get more information. bool isBranch(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Branch, Type); } - /// isIndirectBranch - Return true if this is an indirect branch, such as a + /// Return true if this is an indirect branch, such as a /// branch through a register. bool isIndirectBranch(QueryType Type = AnyInBundle) const { return hasProperty(MCID::IndirectBranch, Type); } - /// isConditionalBranch - Return true if this is a branch which may fall + /// Return true if this is a branch which may fall /// through to the next instruction or may transfer control flow to some other /// block. The TargetInstrInfo::AnalyzeBranch method can be used to get more /// information about this branch. @@ -454,7 +437,7 @@ public: return isBranch(Type) & !isBarrier(Type) & !isIndirectBranch(Type); } - /// isUnconditionalBranch - Return true if this is a branch which always + /// Return true if this is a branch which always /// transfers control flow to some other block. The /// TargetInstrInfo::AnalyzeBranch method can be used to get more information /// about this branch. @@ -472,43 +455,41 @@ public: return hasProperty(MCID::Predicable, Type); } - /// isCompare - Return true if this instruction is a comparison. + /// Return true if this instruction is a comparison. bool isCompare(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::Compare, Type); } - /// isMoveImmediate - Return true if this instruction is a move immediate + /// Return true if this instruction is a move immediate /// (including conditional moves) instruction. bool isMoveImmediate(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::MoveImm, Type); } - /// isBitcast - Return true if this instruction is a bitcast instruction. - /// + /// Return true if this instruction is a bitcast instruction. bool isBitcast(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::Bitcast, Type); } - /// isSelect - Return true if this instruction is a select instruction. - /// + /// Return true if this instruction is a select instruction. bool isSelect(QueryType Type = IgnoreBundle) const { return hasProperty(MCID::Select, Type); } - /// isNotDuplicable - Return true if this instruction cannot be safely - /// duplicated. For example, if the instruction has a unique labels attached + /// Return true if this instruction cannot be safely duplicated. + /// For example, if the instruction has a unique labels attached /// to it, duplicating it would cause multiple definition errors. bool isNotDuplicable(QueryType Type = AnyInBundle) const { return hasProperty(MCID::NotDuplicable, Type); } - /// hasDelaySlot - Returns true if the specified instruction has a delay slot + /// Returns true if the specified instruction has a delay slot /// which must be filled by the code generator. bool hasDelaySlot(QueryType Type = AnyInBundle) const { return hasProperty(MCID::DelaySlot, Type); } - /// canFoldAsLoad - Return true for instructions that can be folded as + /// Return true for instructions that can be folded as /// memory operands in other instructions. The most common use for this /// is instructions that are simple loads from memory that don't modify /// the loaded value in any way, but it can also be used for instructions @@ -567,7 +548,7 @@ public: // Side Effect Analysis //===--------------------------------------------------------------------===// - /// mayLoad - Return true if this instruction could possibly read memory. + /// Return true if this instruction could possibly read memory. /// Instructions with this flag set are not necessarily simple load /// instructions, they may load a value and modify it, for example. bool mayLoad(QueryType Type = AnyInBundle) const { @@ -579,8 +560,7 @@ public: return hasProperty(MCID::MayLoad, Type); } - - /// mayStore - Return true if this instruction could possibly modify memory. + /// Return true if this instruction could possibly modify memory. /// Instructions with this flag set are not necessarily simple store /// instructions, they may store a modified value based on their operands, or /// may not actually modify anything, for example. @@ -593,11 +573,16 @@ public: return hasProperty(MCID::MayStore, Type); } + /// Return true if this instruction could possibly read or modify memory. + bool mayLoadOrStore(QueryType Type = AnyInBundle) const { + return mayLoad(Type) || mayStore(Type); + } + //===--------------------------------------------------------------------===// // Flags that indicate whether an instruction can be modified by a method. //===--------------------------------------------------------------------===// - /// isCommutable - Return true if this may be a 2- or 3-address + /// Return true if this may be a 2- or 3-address /// instruction (of the form "X = op Y, Z, ..."), which produces the same /// result if Y and Z are exchanged. If this flag is set, then the /// TargetInstrInfo::commuteInstruction method may be used to hack on the @@ -611,7 +596,7 @@ public: return hasProperty(MCID::Commutable, Type); } - /// isConvertibleTo3Addr - Return true if this is a 2-address instruction + /// Return true if this is a 2-address instruction /// which can be changed into a 3-address instruction if needed. Doing this /// transformation can be profitable in the register allocator, because it /// means that the instruction can use a 2-address form if possible, but @@ -629,7 +614,7 @@ public: return hasProperty(MCID::ConvertibleTo3Addr, Type); } - /// usesCustomInsertionHook - Return true if this instruction requires + /// Return true if this instruction requires /// custom insertion support when the DAG scheduler is inserting it into a /// machine basic block. If this is true for the instruction, it basically /// means that it is a pseudo instruction used at SelectionDAG time that is @@ -641,7 +626,7 @@ public: return hasProperty(MCID::UsesCustomInserter, Type); } - /// hasPostISelHook - Return true if this instruction requires *adjustment* + /// Return true if this instruction requires *adjustment* /// after instruction selection by calling a target hook. For example, this /// can be used to fill in ARM 's' optional operand depending on whether /// the conditional flag register is used. @@ -649,8 +634,8 @@ public: return hasProperty(MCID::HasPostISelHook, Type); } - /// isRematerializable - Returns true if this instruction is a candidate for - /// remat. This flag is deprecated, please don't use it anymore. If this + /// Returns true if this instruction is a candidate for remat. + /// This flag is deprecated, please don't use it anymore. If this /// flag is set, the isReallyTriviallyReMaterializable() method is called to /// verify the instruction is really rematable. bool isRematerializable(QueryType Type = AllInBundle) const { @@ -659,9 +644,9 @@ public: return hasProperty(MCID::Rematerializable, Type); } - /// isAsCheapAsAMove - Returns true if this instruction has the same cost (or - /// less) than a move instruction. This is useful during certain types of - /// optimizations (e.g., remat during two-address conversion or machine licm) + /// Returns true if this instruction has the same cost (or less) than a move + /// instruction. This is useful during certain types of optimizations + /// (e.g., remat during two-address conversion or machine licm) /// where we would like to remat or hoist the instruction, but not if it costs /// more than moving the instruction into the appropriate register. Note, we /// are not marking copies from and to the same register class with this flag. @@ -670,7 +655,7 @@ public: return hasProperty(MCID::CheapAsAMove, Type); } - /// hasExtraSrcRegAllocReq - Returns true if this instruction source operands + /// Returns true if this instruction source operands /// have special register allocation requirements that are not captured by the /// operand register classes. e.g. ARM::STRD's two source registers must be an /// even / odd pair, ARM::STM registers have to be in ascending order. @@ -680,7 +665,7 @@ public: return hasProperty(MCID::ExtraSrcRegAllocReq, Type); } - /// hasExtraDefRegAllocReq - Returns true if this instruction def operands + /// Returns true if this instruction def operands /// have special register allocation requirements that are not captured by the /// operand register classes. e.g. ARM::LDRD's two def registers must be an /// even / odd pair, ARM::LDM registers have to be in ascending order. @@ -698,7 +683,7 @@ public: IgnoreVRegDefs // Ignore virtual register definitions }; - /// isIdenticalTo - Return true if this instruction is identical to (same + /// Return true if this instruction is identical to (same /// opcode and same operands as) the specified instruction. bool isIdenticalTo(const MachineInstr *Other, MICheckType Check = CheckDefs) const; @@ -739,8 +724,7 @@ public: bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } - /// isLabel - Returns true if the MachineInstr represents a label. - /// + /// Returns true if the MachineInstr represents a label. bool isLabel() const { return isEHLabel() || isGCLabel(); } bool isCFIInstruction() const { return getOpcode() == TargetOpcode::CFI_INSTRUCTION; @@ -789,19 +773,19 @@ public: return getOpcode() == TargetOpcode::EXTRACT_SUBREG; } - /// isCopyLike - Return true if the instruction behaves like a copy. + /// Return true if the instruction behaves like a copy. /// This does not include native copy instructions. bool isCopyLike() const { return isCopy() || isSubregToReg(); } - /// isIdentityCopy - Return true is the instruction is an identity copy. + /// Return true is the instruction is an identity copy. bool isIdentityCopy() const { return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && getOperand(0).getSubReg() == getOperand(1).getSubReg(); } - /// isTransient - Return true if this is a transient instruction that is + /// Return true if this is a transient instruction that is /// either very likely to be eliminated during register allocation (such as /// copy-like instructions), or if this instruction doesn't have an /// execution-time cost. @@ -832,8 +816,8 @@ public: /// skips, 0 for unbundled instructions. unsigned getBundleSize() const; - /// readsRegister - Return true if the MachineInstr reads the specified - /// register. If TargetRegisterInfo is passed, then it also checks if there + /// Return true if the MachineInstr reads the specified register. + /// If TargetRegisterInfo is passed, then it also checks if there /// is a read of a super-register. /// This does not count partial redefines of virtual registers as reads: /// %reg1024:6 = OP. @@ -842,30 +826,29 @@ public: return findRegisterUseOperandIdx(Reg, false, TRI) != -1; } - /// readsVirtualRegister - Return true if the MachineInstr reads the specified - /// virtual register. Take into account that a partial define is a + /// Return true if the MachineInstr reads the specified virtual register. + /// Take into account that a partial define is a /// read-modify-write operation. bool readsVirtualRegister(unsigned Reg) const { return readsWritesVirtualRegister(Reg).first; } - /// readsWritesVirtualRegister - Return a pair of bools (reads, writes) - /// indicating if this instruction reads or writes Reg. This also considers - /// partial defines. + /// Return a pair of bools (reads, writes) indicating if this instruction + /// reads or writes Reg. This also considers partial defines. /// If Ops is not null, all operand indices for Reg are added. std::pair<bool,bool> readsWritesVirtualRegister(unsigned Reg, SmallVectorImpl<unsigned> *Ops = nullptr) const; - /// killsRegister - Return true if the MachineInstr kills the specified - /// register. If TargetRegisterInfo is passed, then it also checks if there is + /// Return true if the MachineInstr kills the specified register. + /// If TargetRegisterInfo is passed, then it also checks if there is /// a kill of a super-register. bool killsRegister(unsigned Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterUseOperandIdx(Reg, true, TRI) != -1; } - /// definesRegister - Return true if the MachineInstr fully defines the - /// specified register. If TargetRegisterInfo is passed, then it also checks + /// Return true if the MachineInstr fully defines the specified register. + /// If TargetRegisterInfo is passed, then it also checks /// if there is a def of a super-register. /// NOTE: It's ignoring subreg indices on virtual registers. bool definesRegister(unsigned Reg, @@ -873,28 +856,28 @@ public: return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; } - /// modifiesRegister - Return true if the MachineInstr modifies (fully define - /// or partially define) the specified register. + /// Return true if the MachineInstr modifies (fully define or partially + /// define) the specified register. /// NOTE: It's ignoring subreg indices on virtual registers. bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const { return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; } - /// registerDefIsDead - Returns true if the register is dead in this machine - /// instruction. If TargetRegisterInfo is passed, then it also checks + /// Returns true if the register is dead in this machine instruction. + /// If TargetRegisterInfo is passed, then it also checks /// if there is a dead def of a super-register. bool registerDefIsDead(unsigned Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; } - /// findRegisterUseOperandIdx() - Returns the operand index that is a use of - /// the specific register or -1 if it is not found. It further tightens - /// the search criteria to a use that kills the register if isKill is true. + /// Returns the operand index that is a use of the specific register or -1 + /// if it is not found. It further tightens the search criteria to a use + /// that kills the register if isKill is true. int findRegisterUseOperandIdx(unsigned Reg, bool isKill = false, const TargetRegisterInfo *TRI = nullptr) const; - /// findRegisterUseOperand - Wrapper for findRegisterUseOperandIdx, it returns + /// Wrapper for findRegisterUseOperandIdx, it returns /// a pointer to the MachineOperand rather than an index. MachineOperand *findRegisterUseOperand(unsigned Reg, bool isKill = false, const TargetRegisterInfo *TRI = nullptr) { @@ -902,17 +885,17 @@ public: return (Idx == -1) ? nullptr : &getOperand(Idx); } - /// findRegisterDefOperandIdx() - Returns the operand index that is a def of - /// the specified register or -1 if it is not found. If isDead is true, defs - /// that are not dead are skipped. If Overlap is true, then it also looks for - /// defs that merely overlap the specified register. If TargetRegisterInfo is - /// non-null, then it also checks if there is a def of a super-register. + /// Returns the operand index that is a def of the specified register or + /// -1 if it is not found. If isDead is true, defs that are not dead are + /// skipped. If Overlap is true, then it also looks for defs that merely + /// overlap the specified register. If TargetRegisterInfo is non-null, + /// then it also checks if there is a def of a super-register. /// This may also return a register mask operand when Overlap is true. int findRegisterDefOperandIdx(unsigned Reg, bool isDead = false, bool Overlap = false, const TargetRegisterInfo *TRI = nullptr) const; - /// findRegisterDefOperand - Wrapper for findRegisterDefOperandIdx, it returns + /// Wrapper for findRegisterDefOperandIdx, it returns /// a pointer to the MachineOperand rather than an index. MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false, const TargetRegisterInfo *TRI = nullptr) { @@ -920,12 +903,12 @@ public: return (Idx == -1) ? nullptr : &getOperand(Idx); } - /// findFirstPredOperandIdx() - Find the index of the first operand in the + /// Find the index of the first operand in the /// operand list that is used to represent the predicate. It returns -1 if /// none is found. int findFirstPredOperandIdx() const; - /// findInlineAsmFlagIdx() - Find the index of the flag word operand that + /// Find the index of the flag word operand that /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if /// getOperand(OpIdx) does not belong to an inline asm operand group. /// @@ -937,9 +920,9 @@ public: /// int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; - /// getRegClassConstraint - Compute the static register class constraint for - /// operand OpIdx. For normal instructions, this is derived from the - /// MCInstrDesc. For inline assembly it is derived from the flag words. + /// Compute the static register class constraint for operand OpIdx. + /// For normal instructions, this is derived from the MCInstrDesc. + /// For inline assembly it is derived from the flag words. /// /// Returns NULL if the static register classs constraint cannot be /// determined. @@ -980,20 +963,20 @@ public: const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; - /// tieOperands - Add a tie between the register operands at DefIdx and - /// UseIdx. The tie will cause the register allocator to ensure that the two + /// Add a tie between the register operands at DefIdx and UseIdx. + /// The tie will cause the register allocator to ensure that the two /// operands are assigned the same physical register. /// /// Tied operands are managed automatically for explicit operands in the /// MCInstrDesc. This method is for exceptional cases like inline asm. void tieOperands(unsigned DefIdx, unsigned UseIdx); - /// findTiedOperandIdx - Given the index of a tied register operand, find the + /// Given the index of a tied register operand, find the /// operand it is tied to. Defs are tied to uses and vice versa. Returns the /// index of the tied operand which must exist. unsigned findTiedOperandIdx(unsigned OpIdx) const; - /// isRegTiedToUseOperand - Given the index of a register def operand, + /// Given the index of a register def operand, /// check if the register def is tied to a source operand, due to either /// two-address elimination or inline assembly constraints. Returns the /// first tied use operand index by reference if UseOpIdx is not null. @@ -1007,9 +990,9 @@ public: return true; } - /// isRegTiedToDefOperand - Return true if the use operand of the specified - /// index is tied to a def operand. It also returns the def operand index by - /// reference if DefOpIdx is not null. + /// Return true if the use operand of the specified index is tied to a def + /// operand. It also returns the def operand index by reference if DefOpIdx + /// is not null. bool isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx = nullptr) const { const MachineOperand &MO = getOperand(UseOpIdx); @@ -1020,16 +1003,15 @@ public: return true; } - /// clearKillInfo - Clears kill flags on all operands. - /// + /// Clears kill flags on all operands. void clearKillInfo(); - /// substituteRegister - Replace all occurrences of FromReg with ToReg:SubIdx, + /// Replace all occurrences of FromReg with ToReg:SubIdx, /// properly composing subreg indices where necessary. void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo); - /// addRegisterKilled - We have determined MI kills a register. Look for the + /// We have determined MI kills a register. Look for the /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, /// add a implicit operand if it's not found. Returns true if the operand /// exists / is added. @@ -1037,23 +1019,31 @@ public: const TargetRegisterInfo *RegInfo, bool AddIfNotFound = false); - /// clearRegisterKills - Clear all kill flags affecting Reg. If RegInfo is + /// Clear all kill flags affecting Reg. If RegInfo is /// provided, this includes super-register kills. void clearRegisterKills(unsigned Reg, const TargetRegisterInfo *RegInfo); - /// addRegisterDead - We have determined MI defined a register without a use. + /// We have determined MI defined a register without a use. /// Look for the operand that defines it and mark it as IsDead. If /// AddIfNotFound is true, add a implicit operand if it's not found. Returns /// true if the operand exists / is added. bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound = false); - /// addRegisterDefined - We have determined MI defines a register. Make sure - /// there is an operand defining Reg. + /// Clear all dead flags on operands defining register @p Reg. + void clearRegisterDeads(unsigned Reg); + + /// Mark all subregister defs of register @p Reg with the undef flag. + /// This function is used when we determined to have a subregister def in an + /// otherwise undefined super register. + void addRegisterDefReadUndef(unsigned Reg); + + /// We have determined MI defines a register. Make sure there is an operand + /// defining Reg. void addRegisterDefined(unsigned Reg, const TargetRegisterInfo *RegInfo = nullptr); - /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as + /// Mark every physreg used by this instruction as /// dead except those in the UsedRegs list. /// /// On instructions with register mask operands, also add implicit-def @@ -1061,32 +1051,30 @@ public: void setPhysRegsDeadExcept(ArrayRef<unsigned> UsedRegs, const TargetRegisterInfo &TRI); - /// isSafeToMove - Return true if it is safe to move this instruction. If + /// Return true if it is safe to move this instruction. If /// SawStore is set to true, it means that there is a store (or call) between /// the instruction's location and its intended destination. - bool isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA, - bool &SawStore) const; + bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const; - /// hasOrderedMemoryRef - Return true if this instruction may have an ordered + /// Return true if this instruction may have an ordered /// or volatile memory reference, or if the information describing the memory /// reference is not available. Return false if it is known to have no /// ordered or volatile memory references. bool hasOrderedMemoryRef() const; - /// isInvariantLoad - Return true if this instruction is loading from a + /// Return true if this instruction is loading from a /// location whose value is invariant across the function. For example, /// loading a value from the constant pool or from the argument area of /// a function if it does not change. This should only return true of *all* /// loads the instruction does are invariant (if it does multiple loads). bool isInvariantLoad(AliasAnalysis *AA) const; - /// isConstantValuePHI - If the specified instruction is a PHI that always - /// merges together the same virtual register, return the register, otherwise - /// return 0. + /// If the specified instruction is a PHI that always merges together the + /// same virtual register, return the register, otherwise return 0. unsigned isConstantValuePHI() const; - /// hasUnmodeledSideEffects - Return true if this instruction has side - /// effects that are not modeled by mayLoad / mayStore, etc. + /// Return true if this instruction has side effects that are not modeled + /// by mayLoad / mayStore, etc. /// For all instructions, the property is encoded in MCInstrDesc::Flags /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is /// INLINEASM instruction, in which case the side effect property is encoded @@ -1094,19 +1082,17 @@ public: /// bool hasUnmodeledSideEffects() const; - /// allDefsAreDead - Return true if all the defs of this instruction are dead. - /// + /// Return true if all the defs of this instruction are dead. bool allDefsAreDead() const; - /// copyImplicitOps - Copy implicit register operands from specified + /// Copy implicit register operands from specified /// instruction to this instruction. void copyImplicitOps(MachineFunction &MF, const MachineInstr *MI); // // Debugging support // - void print(raw_ostream &OS, const TargetMachine *TM = nullptr, - bool SkipOpers = false) const; + void print(raw_ostream &OS, bool SkipOpers = false) const; void dump() const; //===--------------------------------------------------------------------===// @@ -1131,44 +1117,41 @@ public: /// preferred. void addOperand(const MachineOperand &Op); - /// setDesc - Replace the instruction descriptor (thus opcode) of + /// Replace the instruction descriptor (thus opcode) of /// the current instruction with a new one. - /// void setDesc(const MCInstrDesc &tid) { MCID = &tid; } - /// setDebugLoc - Replace current source information with new such. + /// Replace current source information with new such. /// Avoid using this, the constructor argument is preferable. - /// - void setDebugLoc(const DebugLoc dl) { - debugLoc = dl; + void setDebugLoc(DebugLoc dl) { + debugLoc = std::move(dl); assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); } - /// RemoveOperand - Erase an operand from an instruction, leaving it with one + /// Erase an operand from an instruction, leaving it with one /// fewer operand than it started with. - /// void RemoveOperand(unsigned i); - /// addMemOperand - Add a MachineMemOperand to the machine instruction. + /// Add a MachineMemOperand to the machine instruction. /// This function should be used only occasionally. The setMemRefs function /// is the primary method for setting up a MachineInstr's MemRefs list. void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); - /// setMemRefs - Assign this MachineInstr's memory reference descriptor - /// list. This does not transfer ownership. + /// Assign this MachineInstr's memory reference descriptor list. + /// This does not transfer ownership. void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { MemRefs = NewMemRefs; NumMemRefs = uint8_t(NewMemRefsEnd - NewMemRefs); assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs"); } -private: - /// getRegInfo - If this instruction is embedded into a MachineFunction, - /// return the MachineRegisterInfo object for the current function, otherwise - /// return null. - MachineRegisterInfo *getRegInfo(); + /// Clear this MachineInstr's memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; + } - /// untieRegOperand - Break any tie involving OpIdx. + /// Break any tie involving OpIdx. void untieRegOperand(unsigned OpIdx) { MachineOperand &MO = getOperand(OpIdx); if (MO.isReg() && MO.isTied()) { @@ -1177,22 +1160,27 @@ private: } } - /// addImplicitDefUseOperands - Add all implicit def and use operands to - /// this instruction. + +private: + /// If this instruction is embedded into a MachineFunction, return the + /// MachineRegisterInfo object for the current function, otherwise + /// return null. + MachineRegisterInfo *getRegInfo(); + + /// Add all implicit def and use operands to this instruction. void addImplicitDefUseOperands(MachineFunction &MF); - /// RemoveRegOperandsFromUseLists - Unlink all of the register operands in - /// this instruction from their respective use lists. This requires that the - /// operands already be on their use lists. + /// Unlink all of the register operands in this instruction from their + /// respective use lists. This requires that the operands already be on their + /// use lists. void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); - /// AddRegOperandsToUseLists - Add all of the register operands in - /// this instruction from their respective use lists. This requires that the - /// operands not be on their use lists yet. + /// Add all of the register operands in this instruction from their + /// respective use lists. This requires that the operands not be on their + /// use lists yet. void AddRegOperandsToUseLists(MachineRegisterInfo&); - /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a - /// bundle. + /// Slow path for hasProperty when we're dealing with a bundle. bool hasPropertyInBundle(unsigned Mask, QueryType Type) const; /// \brief Implements the logic of getRegClassConstraintEffectForVReg for the @@ -1203,8 +1191,8 @@ private: const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; }; -/// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare -/// MachineInstr* by *value* of the instruction rather than by pointer value. +/// Special DenseMapInfo traits to compare MachineInstr* by *value* of the +/// instruction rather than by pointer value. /// The hashing and equality testing functions ignore definitions so this is /// useful for CSE, etc. struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index 8859b6a019ea4..47397c6d67434 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -174,8 +174,9 @@ public: const MachineInstrBuilder &addMetadata(const MDNode *MD) const { MI->addOperand(*MF, MachineOperand::CreateMetadata(MD)); - assert((MI->isDebugValue() ? MI->getDebugVariable().Verify() : true) && - "first MDNode argument of a DBG_VALUE not a DIVariable"); + assert((MI->isDebugValue() ? static_cast<bool>(MI->getDebugVariable()) + : true) && + "first MDNode argument of a DBG_VALUE not a variable"); return *this; } @@ -355,8 +356,10 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID, bool IsIndirect, unsigned Reg, unsigned Offset, const MDNode *Variable, const MDNode *Expr) { - assert(DIVariable(Variable).Verify() && "not a DIVariable"); - assert(DIExpression(Expr).Verify() && "not a DIExpression"); + assert(isa<DILocalVariable>(Variable) && "not a variable"); + assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); + assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); if (IsIndirect) return BuildMI(MF, DL, MCID) .addReg(Reg, RegState::Debug) @@ -382,8 +385,8 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, const MCInstrDesc &MCID, bool IsIndirect, unsigned Reg, unsigned Offset, const MDNode *Variable, const MDNode *Expr) { - assert(DIVariable(Variable).Verify() && "not a DIVariable"); - assert(DIExpression(Expr).Verify() && "not a DIExpression"); + assert(isa<DILocalVariable>(Variable) && "not a variable"); + assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); MachineFunction &MF = *BB.getParent(); MachineInstr *MI = BuildMI(MF, DL, MCID, IsIndirect, Reg, Offset, Variable, Expr); diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h index 4fbd46b6fb575..f7bcf45d93c56 100644 --- a/include/llvm/CodeGen/MachineLoopInfo.h +++ b/include/llvm/CodeGen/MachineLoopInfo.h @@ -74,8 +74,8 @@ class MachineLoopInfo : public MachineFunctionPass { LoopInfoBase<MachineBasicBlock, MachineLoop> LI; friend class LoopBase<MachineBasicBlock, MachineLoop>; - void operator=(const MachineLoopInfo &) LLVM_DELETED_FUNCTION; - MachineLoopInfo(const MachineLoopInfo &) LLVM_DELETED_FUNCTION; + void operator=(const MachineLoopInfo &) = delete; + MachineLoopInfo(const MachineLoopInfo &) = delete; public: static char ID; // Pass identification, replacement for typeid diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index eb5086cbe5a5f..a16c294a07491 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -199,6 +199,24 @@ public: /// Profile - Gather unique data for the object. /// void Profile(FoldingSetNodeID &ID) const; + + friend bool operator==(const MachineMemOperand &LHS, + const MachineMemOperand &RHS) { + return LHS.getValue() == RHS.getValue() && + LHS.getPseudoValue() == RHS.getPseudoValue() && + LHS.getSize() == RHS.getSize() && + LHS.getOffset() == RHS.getOffset() && + LHS.getFlags() == RHS.getFlags() && + LHS.getAAInfo() == RHS.getAAInfo() && + LHS.getRanges() == RHS.getRanges() && + LHS.getAlignment() == RHS.getAlignment() && + LHS.getAddrSpace() == RHS.getAddrSpace(); + } + + friend bool operator!=(const MachineMemOperand &LHS, + const MachineMemOperand &RHS) { + return !(LHS == RHS); + } }; raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO); diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 584ce65c9d175..ce45c16591533 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -35,6 +35,7 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LibCallSemantics.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" @@ -50,6 +51,7 @@ namespace llvm { // Forward declarations. class Constant; class GlobalVariable; +class BlockAddress; class MDNode; class MMIAddrLabelMap; class MachineBasicBlock; @@ -57,21 +59,33 @@ class MachineFunction; class Module; class PointerType; class StructType; +struct WinEHFuncInfo; + +struct SEHHandler { + // Filter or finally function. Null indicates a catch-all. + const Function *FilterOrFinally; + + // Address of block to recover at. Null for a finally handler. + const BlockAddress *RecoverBA; +}; //===----------------------------------------------------------------------===// /// LandingPadInfo - This structure is used to retain landing pad info for /// the current function. /// struct LandingPadInfo { - MachineBasicBlock *LandingPadBlock; // Landing pad block. - SmallVector<MCSymbol*, 1> BeginLabels; // Labels prior to invoke. - SmallVector<MCSymbol*, 1> EndLabels; // Labels after invoke. - MCSymbol *LandingPadLabel; // Label at beginning of landing pad. - const Function *Personality; // Personality function. - std::vector<int> TypeIds; // List of type ids (filters negative) + MachineBasicBlock *LandingPadBlock; // Landing pad block. + SmallVector<MCSymbol *, 1> BeginLabels; // Labels prior to invoke. + SmallVector<MCSymbol *, 1> EndLabels; // Labels after invoke. + SmallVector<SEHHandler, 1> SEHHandlers; // SEH handlers active at this lpad. + MCSymbol *LandingPadLabel; // Label at beginning of landing pad. + const Function *Personality; // Personality function. + std::vector<int> TypeIds; // List of type ids (filters negative). + int WinEHState; // WinEH specific state number. explicit LandingPadInfo(MachineBasicBlock *MBB) - : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr) {} + : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr), + WinEHState(-1) {} }; //===----------------------------------------------------------------------===// @@ -86,7 +100,10 @@ public: virtual ~MachineModuleInfoImpl(); typedef std::vector<std::pair<MCSymbol*, StubValueTy> > SymbolListTy; protected: - static SymbolListTy GetSortedStubs(const DenseMap<MCSymbol*, StubValueTy>&); + + /// Return the entries from a DenseMap in a deterministic sorted orer. + /// Clears the map. + static SymbolListTy getSortedStubs(DenseMap<MCSymbol*, StubValueTy>&); }; //===----------------------------------------------------------------------===// @@ -168,16 +185,21 @@ class MachineModuleInfo : public ImmutablePass { /// details. bool UsesMorestackAddr; + EHPersonality PersonalityTypeCache; + + DenseMap<const Function *, std::unique_ptr<WinEHFuncInfo>> FuncInfoMap; + public: static char ID; // Pass identification, replacement for typeid struct VariableDbgInfo { - TrackingMDNodeRef Var; - TrackingMDNodeRef Expr; + const DILocalVariable *Var; + const DIExpression *Expr; unsigned Slot; - DebugLoc Loc; + const DILocation *Loc; - VariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot, DebugLoc Loc) + VariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, + unsigned Slot, const DILocation *Loc) : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {} }; typedef SmallVector<VariableDbgInfo, 4> VariableDbgInfoMapTy; @@ -187,7 +209,7 @@ public: // Real constructor. MachineModuleInfo(const MCAsmInfo &MAI, const MCRegisterInfo &MRI, const MCObjectFileInfo *MOFI); - ~MachineModuleInfo(); + ~MachineModuleInfo() override; // Initialization and Finalization bool doInitialization(Module &) override; @@ -203,6 +225,12 @@ public: void setModule(const Module *M) { TheModule = M; } const Module *getModule() const { return TheModule; } + const Function *getWinEHParent(const Function *F) const; + WinEHFuncInfo &getWinEHFuncInfo(const Function *F); + bool hasWinEHFuncInfo(const Function *F) const { + return FuncInfoMap.count(getWinEHParent(F)) > 0; + } + /// getInfo - Keep track of various per-function pieces of information for /// backends that would like to do so. /// @@ -300,6 +328,8 @@ public: void addPersonality(MachineBasicBlock *LandingPad, const Function *Personality); + void addWinEHState(MachineBasicBlock *LandingPad, int State); + /// getPersonalityIndex - Get index of the current personality function inside /// Personalitites array unsigned getPersonalityIndex() const; @@ -330,6 +360,12 @@ public: /// void addCleanup(MachineBasicBlock *LandingPad); + void addSEHCatchHandler(MachineBasicBlock *LandingPad, const Function *Filter, + const BlockAddress *RecoverLabel); + + void addSEHCleanupHandler(MachineBasicBlock *LandingPad, + const Function *Cleanup); + /// getTypeIDFor - Return the type id for the specified typeinfo. This is /// function wide. unsigned getTypeIDFor(const GlobalValue *TI); @@ -407,10 +443,13 @@ public: /// of one is required to emit exception handling info. const Function *getPersonality() const; + /// Classify the personality function amongst known EH styles. + EHPersonality getPersonalityType(); + /// setVariableDbgInfo - Collect information used to emit debugging /// information of a variable. - void setVariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot, - DebugLoc Loc) { + void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, + unsigned Slot, const DILocation *Loc) { VariableDbgInfos.emplace_back(Var, Expr, Slot, Loc); } diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h index 7afc7eb6b3577..a67f9b5666b17 100644 --- a/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -58,14 +58,14 @@ namespace llvm { } /// Accessor methods to return the set of stubs in sorted order. - SymbolListTy GetFnStubList() const { - return GetSortedStubs(FnStubs); + SymbolListTy GetFnStubList() { + return getSortedStubs(FnStubs); } - SymbolListTy GetGVStubList() const { - return GetSortedStubs(GVStubs); + SymbolListTy GetGVStubList() { + return getSortedStubs(GVStubs); } - SymbolListTy GetHiddenGVStubList() const { - return GetSortedStubs(HiddenGVStubs); + SymbolListTy GetHiddenGVStubList() { + return getSortedStubs(HiddenGVStubs); } }; @@ -87,8 +87,8 @@ namespace llvm { /// Accessor methods to return the set of stubs in sorted order. - SymbolListTy GetGVStubList() const { - return GetSortedStubs(GVStubs); + SymbolListTy GetGVStubList() { + return getSortedStubs(GVStubs); } }; diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h index eed1e575f93b6..ddffdcaf1e4db 100644 --- a/include/llvm/CodeGen/MachineOperand.h +++ b/include/llvm/CodeGen/MachineOperand.h @@ -65,7 +65,7 @@ public: private: /// OpKind - Specify what kind of operand this is. This discriminates the /// union. - MachineOperandType OpKind; + MachineOperandType OpKind : 8; /// Subregister number for MO_Register. A value of 0 indicates the /// MO_Register has no subReg. @@ -217,7 +217,7 @@ public: /// void clearParent() { ParentMI = nullptr; } - void print(raw_ostream &os, const TargetMachine *TM = nullptr) const; + void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr) const; //===--------------------------------------------------------------------===// // Accessors that tell you what kind of MachineOperand you're looking at. @@ -554,6 +554,12 @@ public: /// already, the setFPImm method should be used. void ChangeToFPImmediate(const ConstantFP *FPImm); + /// ChangeToES - Replace this operand with a new external symbol operand. + void ChangeToES(const char *SymName, unsigned char TargetFlags = 0); + + /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand. + void ChangeToMCSymbol(MCSymbol *Sym); + /// ChangeToRegister - Replace this operand with a new register operand of /// the specified value. If an operand is known to be an register already, /// the setReg method should be used. diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h index c962e687243e7..6731983c58746 100644 --- a/include/llvm/CodeGen/MachinePassRegistry.h +++ b/include/llvm/CodeGen/MachinePassRegistry.h @@ -122,11 +122,12 @@ template<class RegistryClass> class RegisterPassParser : public MachinePassRegistryListener, public cl::parser<typename RegistryClass::FunctionPassCtor> { public: - RegisterPassParser() {} - ~RegisterPassParser() { RegistryClass::setListener(nullptr); } + RegisterPassParser(cl::Option &O) + : cl::parser<typename RegistryClass::FunctionPassCtor>(O) {} + ~RegisterPassParser() override { RegistryClass::setListener(nullptr); } - void initialize(cl::Option &O) { - cl::parser<typename RegistryClass::FunctionPassCtor>::initialize(O); + void initialize() { + cl::parser<typename RegistryClass::FunctionPassCtor>::initialize(); // Add existing passes to option. for (RegistryClass *Node = RegistryClass::getList(); diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h index aab5c407629f7..70bdb191ad343 100644 --- a/include/llvm/CodeGen/MachinePostDominators.h +++ b/include/llvm/CodeGen/MachinePostDominators.h @@ -33,7 +33,7 @@ public: MachinePostDominatorTree(); - ~MachinePostDominatorTree(); + ~MachinePostDominatorTree() override; FunctionPass *createMachinePostDominatorTreePass(); diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h index 43499dba71c02..cf49c297c2880 100644 --- a/include/llvm/CodeGen/MachineRegionInfo.h +++ b/include/llvm/CodeGen/MachineRegionInfo.h @@ -57,8 +57,6 @@ public: } - ~MachineRegionNode() { } - bool operator==(const MachineRegion &RN) const { return this == reinterpret_cast<const MachineRegionNode*>(&RN); } @@ -80,7 +78,7 @@ class MachineRegionInfo : public RegionInfoBase<RegionTraits<MachineFunction>> { public: explicit MachineRegionInfo(); - virtual ~MachineRegionInfo(); + ~MachineRegionInfo() override; // updateStatistics - Update statistic about created regions. void updateStatistics(MachineRegion *R) final; @@ -98,7 +96,7 @@ public: static char ID; explicit MachineRegionInfoPass(); - ~MachineRegionInfoPass(); + ~MachineRegionInfoPass() override; MachineRegionInfo &getRegionInfo() { return RI; diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index caa48a5cf0cfc..e5b837aeea286 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -123,8 +123,8 @@ private: /// second element. std::vector<std::pair<unsigned, unsigned> > LiveIns; - MachineRegisterInfo(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION; - void operator=(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION; + MachineRegisterInfo(const MachineRegisterInfo&) = delete; + void operator=(const MachineRegisterInfo&) = delete; public: explicit MachineRegisterInfo(const MachineFunction *MF); @@ -182,7 +182,18 @@ public: /// information. void invalidateLiveness() { TracksLiveness = false; } - bool tracksSubRegLiveness() const { return TracksSubRegLiveness; } + /// Returns true if liveness for register class @p RC should be tracked at + /// the subregister level. + bool shouldTrackSubRegLiveness(const TargetRegisterClass &RC) const { + return subRegLivenessEnabled() && RC.HasDisjunctSubRegs; + } + bool shouldTrackSubRegLiveness(unsigned VReg) const { + assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Must pass a VReg"); + return shouldTrackSubRegLiveness(*getRegClass(VReg)); + } + bool subRegLivenessEnabled() const { + return TracksSubRegLiveness; + } void enableSubRegLiveness(bool Enable = true) { TracksSubRegLiveness = Enable; @@ -593,7 +604,7 @@ public: /// virtual register, for example after removing instructions or splitting /// the live range. /// - bool recomputeRegClass(unsigned Reg, const TargetMachine&); + bool recomputeRegClass(unsigned Reg); /// createVirtualRegister - Create and return a new virtual register in the /// function with the specified register class. @@ -609,22 +620,25 @@ public: /// setRegAllocationHint - Specify a register allocation hint for the /// specified virtual register. - void setRegAllocationHint(unsigned Reg, unsigned Type, unsigned PrefReg) { - RegAllocHints[Reg].first = Type; - RegAllocHints[Reg].second = PrefReg; + void setRegAllocationHint(unsigned VReg, unsigned Type, unsigned PrefReg) { + assert(TargetRegisterInfo::isVirtualRegister(VReg)); + RegAllocHints[VReg].first = Type; + RegAllocHints[VReg].second = PrefReg; } /// getRegAllocationHint - Return the register allocation hint for the /// specified virtual register. std::pair<unsigned, unsigned> - getRegAllocationHint(unsigned Reg) const { - return RegAllocHints[Reg]; + getRegAllocationHint(unsigned VReg) const { + assert(TargetRegisterInfo::isVirtualRegister(VReg)); + return RegAllocHints[VReg]; } /// getSimpleHint - Return the preferred register allocation hint, or 0 if a /// standard simple hint (Type == 0) is not set. - unsigned getSimpleHint(unsigned Reg) const { - std::pair<unsigned, unsigned> Hint = getRegAllocationHint(Reg); + unsigned getSimpleHint(unsigned VReg) const { + assert(TargetRegisterInfo::isVirtualRegister(VReg)); + std::pair<unsigned, unsigned> Hint = getRegAllocationHint(VReg); return Hint.first ? 0 : Hint.second; } @@ -829,7 +843,6 @@ public: typedef std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t>::pointer pointer; - defusechain_iterator(const defusechain_iterator &I) : Op(I.Op) {} defusechain_iterator() : Op(nullptr) {} bool operator==(const defusechain_iterator &x) const { @@ -932,7 +945,6 @@ public: typedef std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t>::pointer pointer; - defusechain_instr_iterator(const defusechain_instr_iterator &I) : Op(I.Op){} defusechain_instr_iterator() : Op(nullptr) {} bool operator==(const defusechain_instr_iterator &x) const { diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h index 486a26e927148..5f988ad86320f 100644 --- a/include/llvm/CodeGen/MachineSSAUpdater.h +++ b/include/llvm/CodeGen/MachineSSAUpdater.h @@ -107,8 +107,8 @@ public: private: unsigned GetValueAtEndOfBlockInternal(MachineBasicBlock *BB); - void operator=(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION; - MachineSSAUpdater(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION; + void operator=(const MachineSSAUpdater&) = delete; + MachineSSAUpdater(const MachineSSAUpdater&) = delete; }; } // End llvm namespace diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index a31940161ca52..e80e14e5ccf75 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -385,7 +385,7 @@ public: ShouldTrackPressure(false), RPTracker(RegPressure), TopRPTracker(TopPressure), BotRPTracker(BotPressure) {} - virtual ~ScheduleDAGMILive(); + ~ScheduleDAGMILive() override; /// Return true if this DAG supports VReg liveness and RegPressure. bool hasVRegLiveness() const override { return true; } @@ -909,7 +909,7 @@ public: PostGenericScheduler(const MachineSchedContext *C): GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ") {} - virtual ~PostGenericScheduler() {} + ~PostGenericScheduler() override {} void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h index e3fbfe89c203e..a728df3546778 100644 --- a/include/llvm/CodeGen/MachineValueType.h +++ b/include/llvm/CodeGen/MachineValueType.h @@ -26,7 +26,7 @@ namespace llvm { /// MVT - Machine Value Type. Every type that is supported natively by some /// processor targeted by LLVM occurs here. This means that any legal value /// type can be represented by an MVT. - class MVT { +class MVT { public: enum SimpleValueType { // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are @@ -86,22 +86,23 @@ namespace llvm { v4i64 = 39, // 4 x i64 v8i64 = 40, // 8 x i64 v16i64 = 41, // 16 x i64 - + v1i128 = 42, // 1 x i128 + FIRST_INTEGER_VECTOR_VALUETYPE = v2i1, - LAST_INTEGER_VECTOR_VALUETYPE = v16i64, - - v2f16 = 42, // 2 x f16 - v4f16 = 43, // 4 x f16 - v8f16 = 44, // 8 x f16 - v1f32 = 45, // 1 x f32 - v2f32 = 46, // 2 x f32 - v4f32 = 47, // 4 x f32 - v8f32 = 48, // 8 x f32 - v16f32 = 49, // 16 x f32 - v1f64 = 50, // 1 x f64 - v2f64 = 51, // 2 x f64 - v4f64 = 52, // 4 x f64 - v8f64 = 53, // 8 x f64 + LAST_INTEGER_VECTOR_VALUETYPE = v1i128, + + v2f16 = 43, // 2 x f16 + v4f16 = 44, // 4 x f16 + v8f16 = 45, // 8 x f16 + v1f32 = 46, // 1 x f32 + v2f32 = 47, // 2 x f32 + v4f32 = 48, // 4 x f32 + v8f32 = 49, // 8 x f32 + v16f32 = 50, // 16 x f32 + v1f64 = 51, // 1 x f64 + v2f64 = 52, // 2 x f64 + v4f64 = 53, // 4 x f64 + v8f64 = 54, // 8 x f64 FIRST_FP_VECTOR_VALUETYPE = v2f16, LAST_FP_VECTOR_VALUETYPE = v8f64, @@ -109,18 +110,18 @@ namespace llvm { FIRST_VECTOR_VALUETYPE = v2i1, LAST_VECTOR_VALUETYPE = v8f64, - x86mmx = 54, // This is an X86 MMX value + x86mmx = 55, // This is an X86 MMX value - Glue = 55, // This glues nodes together during pre-RA sched + Glue = 56, // This glues nodes together during pre-RA sched - isVoid = 56, // This has no value + isVoid = 57, // This has no value - Untyped = 57, // This value takes a register, but has + Untyped = 58, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. FIRST_VALUETYPE = 0, // This is always the beginning of the list. - LAST_VALUETYPE = 58, // This always remains at the end of the list. + LAST_VALUETYPE = 59, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors @@ -152,13 +153,17 @@ namespace llvm { // iPTR - An int value the size of the pointer of the current // target. This should only be used internal to tblgen! - iPTR = 255 + iPTR = 255, + + // Any - Any type. This is used for intrinsics that have overloadings. + // This is only for tblgen's consumption! + Any = 256 }; SimpleValueType SimpleTy; - MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {} - MVT(SimpleValueType SVT) : SimpleTy(SVT) { } + LLVM_CONSTEXPR MVT() : SimpleTy(INVALID_SIMPLE_VALUE_TYPE) {} + LLVM_CONSTEXPR MVT(SimpleValueType SVT) : SimpleTy(SVT) { } bool operator>(const MVT& S) const { return SimpleTy > S.SimpleTy; } bool operator<(const MVT& S) const { return SimpleTy < S.SimpleTy; } @@ -218,10 +223,10 @@ namespace llvm { /// is128BitVector - Return true if this is a 128-bit vector type. bool is128BitVector() const { - return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || - SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || - SimpleTy == MVT::v8f16 || SimpleTy == MVT::v4f32 || - SimpleTy == MVT::v2f64); + return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || + SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || + SimpleTy == MVT::v1i128 || SimpleTy == MVT::v8f16 || + SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64); } /// is256BitVector - Return true if this is a 256-bit vector type. @@ -245,7 +250,8 @@ namespace llvm { /// isOverloaded - Return true if this is an overloaded type for TableGen. bool isOverloaded() const { - return (SimpleTy==MVT::iAny || SimpleTy==MVT::fAny || + return (SimpleTy==MVT::Any || + SimpleTy==MVT::iAny || SimpleTy==MVT::fAny || SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny); } @@ -305,6 +311,7 @@ namespace llvm { case v4i64: case v8i64: case v16i64: return i64; + case v1i128: return i128; case v2f16: case v4f16: case v8f16: return f16; @@ -363,6 +370,7 @@ namespace llvm { case v1i16: case v1i32: case v1i64: + case v1i128: case v1f32: case v1f64: return 1; } @@ -380,6 +388,7 @@ namespace llvm { case iAny: case fAny: case vAny: + case Any: llvm_unreachable("Value type is overloaded."); case Metadata: llvm_unreachable("Value type is metadata."); @@ -421,6 +430,7 @@ namespace llvm { case v8i16: case v4i32: case v2i64: + case v1i128: case v8f16: case v4f32: case v2f64: return 128; @@ -556,6 +566,9 @@ namespace llvm { if (NumElements == 8) return MVT::v8i64; if (NumElements == 16) return MVT::v16i64; break; + case MVT::i128: + if (NumElements == 1) return MVT::v1i128; + break; case MVT::f16: if (NumElements == 2) return MVT::v2f16; if (NumElements == 4) return MVT::v4f16; diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h index 4dc5674ae134c..f73383ed10007 100644 --- a/include/llvm/CodeGen/PBQP/Graph.h +++ b/include/llvm/CodeGen/PBQP/Graph.h @@ -21,6 +21,7 @@ #include <list> #include <map> #include <set> +#include <vector> namespace llvm { namespace PBQP { @@ -507,14 +508,14 @@ namespace PBQP { return getNode(NId).getAdjEdgeIds().size(); } - /// @brief Set an edge's cost matrix. + /// @brief Update an edge's cost matrix. /// @param EId Edge id. /// @param Costs New cost matrix. template <typename OtherMatrixT> - void setEdgeCosts(EdgeId EId, OtherMatrixT Costs) { + void updateEdgeCosts(EdgeId EId, OtherMatrixT Costs) { MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs)); if (Solver) - Solver->handleSetEdgeCosts(EId, *AllocatedCosts); + Solver->handleUpdateCosts(EId, *AllocatedCosts); getEdge(EId).Costs = AllocatedCosts; } @@ -548,14 +549,14 @@ namespace PBQP { /// @brief Get the first node connected to this edge. /// @param EId Edge id. /// @return The first node connected to the given edge. - NodeId getEdgeNode1Id(EdgeId EId) { + NodeId getEdgeNode1Id(EdgeId EId) const { return getEdge(EId).getN1Id(); } /// @brief Get the second node connected to this edge. /// @param EId Edge id. /// @return The second node connected to the given edge. - NodeId getEdgeNode2Id(EdgeId EId) { + NodeId getEdgeNode2Id(EdgeId EId) const { return getEdge(EId).getN2Id(); } @@ -672,69 +673,6 @@ namespace PBQP { Edges.clear(); FreeEdgeIds.clear(); } - - /// @brief Dump a graph to an output stream. - template <typename OStream> - void dumpToStream(OStream &OS) { - OS << nodeIds().size() << " " << edgeIds().size() << "\n"; - - for (auto NId : nodeIds()) { - const Vector& V = getNodeCosts(NId); - OS << "\n" << V.getLength() << "\n"; - assert(V.getLength() != 0 && "Empty vector in graph."); - OS << V[0]; - for (unsigned i = 1; i < V.getLength(); ++i) { - OS << " " << V[i]; - } - OS << "\n"; - } - - for (auto EId : edgeIds()) { - NodeId N1Id = getEdgeNode1Id(EId); - NodeId N2Id = getEdgeNode2Id(EId); - assert(N1Id != N2Id && "PBQP graphs shound not have self-edges."); - const Matrix& M = getEdgeCosts(EId); - OS << "\n" << N1Id << " " << N2Id << "\n" - << M.getRows() << " " << M.getCols() << "\n"; - assert(M.getRows() != 0 && "No rows in matrix."); - assert(M.getCols() != 0 && "No cols in matrix."); - for (unsigned i = 0; i < M.getRows(); ++i) { - OS << M[i][0]; - for (unsigned j = 1; j < M.getCols(); ++j) { - OS << " " << M[i][j]; - } - OS << "\n"; - } - } - } - - /// @brief Dump this graph to dbgs(). - void dump() { - dumpToStream(dbgs()); - } - - /// @brief Print a representation of this graph in DOT format. - /// @param OS Output stream to print on. - template <typename OStream> - void printDot(OStream &OS) { - OS << "graph {\n"; - for (auto NId : nodeIds()) { - OS << " node" << NId << " [ label=\"" - << NId << ": " << getNodeCosts(NId) << "\" ]\n"; - } - OS << " edge [ len=" << nodeIds().size() << " ]\n"; - for (auto EId : edgeIds()) { - OS << " node" << getEdgeNode1Id(EId) - << " -- node" << getEdgeNode2Id(EId) - << " [ label=\""; - const Matrix &EdgeCosts = getEdgeCosts(EId); - for (unsigned i = 0; i < EdgeCosts.getRows(); ++i) { - OS << EdgeCosts.getRowAsVector(i) << "\\n"; - } - OS << "\" ]\n"; - } - OS << "}\n"; - } }; } // namespace PBQP diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h index 21fde4d8a5cd1..d4a544bfe7214 100644 --- a/include/llvm/CodeGen/PBQP/ReductionRules.h +++ b/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -132,9 +132,9 @@ namespace PBQP { } else { const Matrix &YZECosts = G.getEdgeCosts(YZEId); if (YNId == G.getEdgeNode1Id(YZEId)) { - G.setEdgeCosts(YZEId, Delta + YZECosts); + G.updateEdgeCosts(YZEId, Delta + YZECosts); } else { - G.setEdgeCosts(YZEId, Delta.transpose() + YZECosts); + G.updateEdgeCosts(YZEId, Delta.transpose() + YZECosts); } } @@ -144,6 +144,25 @@ namespace PBQP { // TODO: Try to normalize newly added/modified edge. } +#ifndef NDEBUG + // Does this Cost vector have any register options ? + template <typename VectorT> + bool hasRegisterOptions(const VectorT &V) { + unsigned VL = V.getLength(); + + // An empty or spill only cost vector does not provide any register option. + if (VL <= 1) + return false; + + // If there are registers in the cost vector, but all of them have infinite + // costs, then ... there is no available register. + for (unsigned i = 1; i < VL; ++i) + if (V[i] != std::numeric_limits<PBQP::PBQPNum>::infinity()) + return true; + + return false; + } +#endif // \brief Find a solution to a fully reduced graph by backpropagation. // @@ -170,6 +189,15 @@ namespace PBQP { RawVector v = G.getNodeCosts(NId); +#ifndef NDEBUG + // Although a conservatively allocatable node can be allocated to a register, + // spilling it may provide a lower cost solution. Assert here that spilling + // is done by choice, not because there were no register available. + if (G.getNodeMetadata(NId).wasConservativelyAllocatable()) + assert(hasRegisterOptions(v) && "A conservatively allocatable node " + "must have available register options"); +#endif + for (auto EId : G.adjEdgeIds(NId)) { const Matrix& edgeCosts = G.getEdgeCosts(EId); if (NId == G.getEdgeNode1Id(EId)) { diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index 8ed32b8a8dd55..39f69549d7cf3 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -120,12 +120,15 @@ protected: /// Default setting for -enable-tail-merge on this target. bool EnableTailMerge; + /// Default setting for -enable-shrink-wrap on this target. + bool EnableShrinkWrap; + public: TargetPassConfig(TargetMachine *tm, PassManagerBase &pm); // Dummy constructor. TargetPassConfig(); - virtual ~TargetPassConfig(); + ~TargetPassConfig() override; static char ID; @@ -179,6 +182,9 @@ public: /// Return true if the optimized regalloc pipeline is enabled. bool getOptimizeRegAlloc() const; + /// Return true if shrink wrapping is enabled. + bool getEnableShrinkWrap() const; + /// Return true if the default global register allocator is in use and /// has not be overriden on the command line with '-regalloc=...' bool usingDefaultRegAlloc() const; @@ -214,7 +220,7 @@ public: /// /// This can also be used to plug a new MachineSchedStrategy into an instance /// of the standard ScheduleDAGMI: - /// return new ScheduleDAGMI(C, new MyStrategy(C)) + /// return new ScheduleDAGMI(C, make_unique<MyStrategy>(C), /* IsPostRA= */false) /// /// Return NULL to select the default (generic) machine scheduler. virtual ScheduleDAGInstrs * @@ -354,13 +360,6 @@ protected: namespace llvm { FunctionPass *createAtomicExpandPass(const TargetMachine *TM); - /// \brief Create a basic TargetTransformInfo analysis pass. - /// - /// This pass implements the target transform info analysis using the target - /// independent information available to the LLVM code generator. - ImmutablePass * - createBasicTargetTransformInfoPass(const TargetMachine *TM); - /// createUnreachableBlockEliminationPass - The LLVM code generator does not /// work well with unreachable basic blocks (what live ranges make sense for a /// block that cannot be reached?). As such, a code generator should either @@ -433,6 +432,10 @@ namespace llvm { /// basic blocks. extern char &SpillPlacementID; + /// ShrinkWrap pass. Look for the best place to insert save and restore + // instruction and update the MachineFunctionInfo with that information. + extern char &ShrinkWrapID; + /// VirtRegRewriter pass. Rewrite virtual registers to physical registers as /// assigned in VirtRegMap. extern char &VirtRegRewriterID; @@ -517,11 +520,15 @@ namespace llvm { /// information. extern char &MachineBlockPlacementStatsID; - /// GCLowering Pass - Performs target-independent LLVM IR transformations for - /// highly portable strategies. - /// + /// GCLowering Pass - Used by gc.root to perform its default lowering + /// operations. FunctionPass *createGCLoweringPass(); + /// ShadowStackGCLowering - Implements the custom lowering mechanism + /// used by the shadow stack GC. Only runs on functions which opt in to + /// the shadow stack collector. + FunctionPass *createShadowStackGCLoweringPass(); + /// GCMachineCodeAnalysis - Target-independent pass to mark safe points /// in machine code. Must be added very late during code generation, just /// prior to output, and importantly after all CFG transformations (such as @@ -569,6 +576,10 @@ namespace llvm { /// adapted to code generation. Required if using dwarf exception handling. FunctionPass *createDwarfEHPass(const TargetMachine *TM); + /// createWinEHPass - Prepares personality functions used by MSVC on Windows, + /// in addition to the Itanium LSDA based personalities. + FunctionPass *createWinEHPass(const TargetMachine *TM); + /// createSjLjEHPreparePass - This pass adapts exception handling code to use /// the GCC-style builtin setjmp/longjmp (sjlj) to handling EH control flow. /// @@ -611,13 +622,13 @@ namespace llvm { ModulePass *createForwardControlFlowIntegrityPass(); } // End llvm namespace -/// This initializer registers TargetMachine constructor, so the pass being -/// initialized can use target dependent interfaces. Please do not move this -/// macro to be together with INITIALIZE_PASS, which is a complete target -/// independent initializer, and we don't want to make libScalarOpts depend -/// on libCodeGen. -#define INITIALIZE_TM_PASS(passName, arg, name, cfg, analysis) \ - static void* initialize##passName##PassOnce(PassRegistry &Registry) { \ +/// Target machine pass initializer for passes with dependencies. Use with +/// INITIALIZE_TM_PASS_END. +#define INITIALIZE_TM_PASS_BEGIN INITIALIZE_PASS_BEGIN + +/// Target machine pass initializer for passes with dependencies. Use with +/// INITIALIZE_TM_PASS_BEGIN. +#define INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis) \ PassInfo *PI = new PassInfo(name, arg, & passName ::ID, \ PassInfo::NormalCtor_t(callDefaultCtor< passName >), cfg, analysis, \ PassInfo::TargetMachineCtor_t(callTargetMachineCtor< passName >)); \ @@ -628,4 +639,13 @@ namespace llvm { CALL_ONCE_INITIALIZATION(initialize##passName##PassOnce) \ } +/// This initializer registers TargetMachine constructor, so the pass being +/// initialized can use target dependent interfaces. Please do not move this +/// macro to be together with INITIALIZE_PASS, which is a complete target +/// independent initializer, and we don't want to make libScalarOpts depend +/// on libCodeGen. +#define INITIALIZE_TM_PASS(passName, arg, name, cfg, analysis) \ + INITIALIZE_TM_PASS_BEGIN(passName, arg, name, cfg, analysis) \ + INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis) + #endif diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h index 5c0e9859915f9..6046e46547b20 100644 --- a/include/llvm/CodeGen/RegAllocPBQP.h +++ b/include/llvm/CodeGen/RegAllocPBQP.h @@ -23,6 +23,9 @@ #include "llvm/Support/ErrorHandling.h" namespace llvm { + +class raw_ostream; + namespace PBQP { namespace RegAlloc { @@ -177,21 +180,34 @@ class NodeMetadata { public: typedef RegAlloc::AllowedRegVector AllowedRegVector; - typedef enum { Unprocessed, - OptimallyReducible, - ConservativelyAllocatable, - NotProvablyAllocatable } ReductionState; + // The node's reduction state. The order in this enum is important, + // as it is assumed nodes can only progress up (i.e. towards being + // optimally reducible) when reducing the graph. + typedef enum { + Unprocessed, + NotProvablyAllocatable, + ConservativelyAllocatable, + OptimallyReducible + } ReductionState; NodeMetadata() : RS(Unprocessed), NumOpts(0), DeniedOpts(0), OptUnsafeEdges(nullptr), - VReg(0) {} + VReg(0) +#ifndef NDEBUG + , everConservativelyAllocatable(false) +#endif + {} // FIXME: Re-implementing default behavior to work around MSVC. Remove once // MSVC synthesizes move constructors properly. NodeMetadata(const NodeMetadata &Other) : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg), - AllowedRegs(Other.AllowedRegs) { + AllowedRegs(Other.AllowedRegs) +#ifndef NDEBUG + , everConservativelyAllocatable(Other.everConservativelyAllocatable) +#endif + { if (NumOpts > 0) { std::copy(&Other.OptUnsafeEdges[0], &Other.OptUnsafeEdges[NumOpts], &OptUnsafeEdges[0]); @@ -203,7 +219,11 @@ public: NodeMetadata(NodeMetadata &&Other) : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), OptUnsafeEdges(std::move(Other.OptUnsafeEdges)), VReg(Other.VReg), - AllowedRegs(std::move(Other.AllowedRegs)) {} + AllowedRegs(std::move(Other.AllowedRegs)) +#ifndef NDEBUG + , everConservativelyAllocatable(Other.everConservativelyAllocatable) +#endif + {} // FIXME: Re-implementing default behavior to work around MSVC. Remove once // MSVC synthesizes move constructors properly. @@ -216,6 +236,9 @@ public: OptUnsafeEdges.get()); VReg = Other.VReg; AllowedRegs = Other.AllowedRegs; +#ifndef NDEBUG + everConservativelyAllocatable = Other.everConservativelyAllocatable; +#endif return *this; } @@ -228,6 +251,9 @@ public: OptUnsafeEdges = std::move(Other.OptUnsafeEdges); VReg = Other.VReg; AllowedRegs = std::move(Other.AllowedRegs); +#ifndef NDEBUG + everConservativelyAllocatable = Other.everConservativelyAllocatable; +#endif return *this; } @@ -245,7 +271,18 @@ public: } ReductionState getReductionState() const { return RS; } - void setReductionState(ReductionState RS) { this->RS = RS; } + void setReductionState(ReductionState RS) { + assert(RS >= this->RS && "A node's reduction state can not be downgraded"); + this->RS = RS; + +#ifndef NDEBUG + // Remember this state to assert later that a non-infinite register + // option was available. + if (RS == ConservativelyAllocatable) + everConservativelyAllocatable = true; +#endif + } + void handleAddEdge(const MatrixMetadata& MD, bool Transpose) { DeniedOpts += Transpose ? MD.getWorstRow() : MD.getWorstCol(); @@ -269,6 +306,12 @@ public: &OptUnsafeEdges[NumOpts]); } +#ifndef NDEBUG + bool wasConservativelyAllocatable() const { + return everConservativelyAllocatable; + } +#endif + private: ReductionState RS; unsigned NumOpts; @@ -276,6 +319,10 @@ private: std::unique_ptr<unsigned[]> OptUnsafeEdges; unsigned VReg; GraphMetadata::AllowedRegVecRef AllowedRegs; + +#ifndef NDEBUG + bool everConservativelyAllocatable; +#endif }; class RegAllocSolverImpl { @@ -309,6 +356,8 @@ public: } void handleAddNode(NodeId NId) { + assert(G.getNodeCosts(NId).getLength() > 1 && + "PBQP Graph should not contain single or zero-option nodes"); G.getNodeMetadata(NId).setup(G.getNodeCosts(NId)); } void handleRemoveNode(NodeId NId) {} @@ -328,15 +377,7 @@ public: NodeMetadata& NMd = G.getNodeMetadata(NId); const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId)); - if (G.getNodeDegree(NId) == 3) { - // This node is becoming optimally reducible. - moveToOptimallyReducibleNodes(NId); - } else if (NMd.getReductionState() == - NodeMetadata::NotProvablyAllocatable && - NMd.isConservativelyAllocatable()) { - // This node just became conservatively allocatable. - moveToConservativelyAllocatableNodes(NId); - } + promote(NId, NMd); } void handleReconnectEdge(EdgeId EId, NodeId NId) { @@ -345,20 +386,44 @@ public: NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId)); } - void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) { - handleRemoveEdge(EId); - + void handleUpdateCosts(EdgeId EId, const Matrix& NewCosts) { NodeId N1Id = G.getEdgeNode1Id(EId); NodeId N2Id = G.getEdgeNode2Id(EId); NodeMetadata& N1Md = G.getNodeMetadata(N1Id); NodeMetadata& N2Md = G.getNodeMetadata(N2Id); + bool Transpose = N1Id != G.getEdgeNode1Id(EId); + + // Metadata are computed incrementally. First, update them + // by removing the old cost. + const MatrixMetadata& OldMMd = G.getEdgeCosts(EId).getMetadata(); + N1Md.handleRemoveEdge(OldMMd, Transpose); + N2Md.handleRemoveEdge(OldMMd, !Transpose); + + // And update now the metadata with the new cost. const MatrixMetadata& MMd = NewCosts.getMetadata(); - N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId)); - N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId)); + N1Md.handleAddEdge(MMd, Transpose); + N2Md.handleAddEdge(MMd, !Transpose); + + // As the metadata may have changed with the update, the nodes may have + // become ConservativelyAllocatable or OptimallyReducible. + promote(N1Id, N1Md); + promote(N2Id, N2Md); } private: + void promote(NodeId NId, NodeMetadata& NMd) { + if (G.getNodeDegree(NId) == 3) { + // This node is becoming optimally reducible. + moveToOptimallyReducibleNodes(NId); + } else if (NMd.getReductionState() == + NodeMetadata::NotProvablyAllocatable && + NMd.isConservativelyAllocatable()) { + // This node just became conservatively allocatable. + moveToConservativelyAllocatableNodes(NId); + } + } + void removeFromCurrentSet(NodeId NId) { switch (G.getNodeMetadata(NId).getReductionState()) { case NodeMetadata::Unprocessed: break; @@ -479,8 +544,10 @@ private: public: SpillCostComparator(const Graph& G) : G(G) {} bool operator()(NodeId N1Id, NodeId N2Id) { - PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id); - PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id); + PBQPNum N1SC = G.getNodeCosts(N1Id)[0]; + PBQPNum N2SC = G.getNodeCosts(N2Id)[0]; + if (N1SC == N2SC) + return G.getNodeDegree(N1Id) < G.getNodeDegree(N2Id); return N1SC < N2SC; } private: @@ -499,6 +566,17 @@ private: typedef PBQP::Graph<RegAllocSolverImpl> BaseT; public: PBQPRAGraph(GraphMetadata Metadata) : BaseT(Metadata) {} + + /// @brief Dump this graph to dbgs(). + void dump() const; + + /// @brief Dump this graph to an output stream. + /// @param OS Output stream to print on. + void dump(raw_ostream &OS) const; + + /// @brief Print a representation of this graph in DOT format. + /// @param OS Output stream to print on. + void printDot(raw_ostream &OS) const; }; inline Solution solve(PBQPRAGraph& G) { diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index cc9e000551464..fcb6feed68ca2 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -35,21 +35,6 @@ struct RegisterPressure { SmallVector<unsigned,8> LiveInRegs; SmallVector<unsigned,8> LiveOutRegs; - /// Increase register pressure for each pressure set impacted by this register - /// class. Normally called by RegPressureTracker, but may be called manually - /// to account for live through (global liveness). - /// - /// \param Reg is either a virtual register number or register unit number. - void increase(unsigned Reg, const TargetRegisterInfo *TRI, - const MachineRegisterInfo *MRI); - - /// Decrease register pressure for each pressure set impacted by this register - /// class. This is only useful to account for spilling or rematerialization. - /// - /// \param Reg is either a virtual register number or register unit number. - void decrease(unsigned Reg, const TargetRegisterInfo *TRI, - const MachineRegisterInfo *MRI); - void dump(const TargetRegisterInfo *TRI) const; }; diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h index 474861e45df1c..df3fd34e0af67 100644 --- a/include/llvm/CodeGen/RegisterScavenging.h +++ b/include/llvm/CodeGen/RegisterScavenging.h @@ -9,7 +9,7 @@ // // This file declares the machine register scavenger class. It can provide // information such as unused register at any point in a machine basic block. -// It also provides a mechanism to make registers availbale by evicting them +// It also provides a mechanism to make registers available by evicting them // to spill slots. // //===----------------------------------------------------------------------===// @@ -36,8 +36,7 @@ class RegScavenger { MachineBasicBlock::iterator MBBI; unsigned NumRegUnits; - /// Tracking - True if RegScavenger is currently tracking the liveness of - /// registers. + /// True if RegScavenger is currently tracking the liveness of registers. bool Tracking; /// Information on scavenged registers (held in a spill slot). @@ -58,8 +57,8 @@ class RegScavenger { /// A vector of information on scavenged registers. SmallVector<ScavengedInfo, 2> Scavenged; - /// RegUnitsAvailable - The current state of each reg unit immediatelly - /// before MBBI. One bit per register unit. If bit is not set it means any + /// The current state of each reg unit immediately before MBBI. + /// One bit per register unit. If bit is not set it means any /// register containing that register unit is currently being used. BitVector RegUnitsAvailable; @@ -72,18 +71,17 @@ public: RegScavenger() : MBB(nullptr), NumRegUnits(0), Tracking(false) {} - /// enterBasicBlock - Start tracking liveness from the begin of the specific - /// basic block. + /// Start tracking liveness from the begin of the specific basic block. void enterBasicBlock(MachineBasicBlock *mbb); - /// initRegState - allow resetting register state info for multiple + /// Allow resetting register state info for multiple /// passes over/within the same function. void initRegState(); - /// forward - Move the internal MBB iterator and update register states. + /// Move the internal MBB iterator and update register states. void forward(); - /// forward - Move the internal MBB iterator and update register states until + /// Move the internal MBB iterator and update register states until /// it has processed the specific iterator. void forward(MachineBasicBlock::iterator I) { if (!Tracking && MBB->begin() != I) forward(); @@ -99,7 +97,7 @@ public: while (MBBI != I) unprocess(); } - /// skipTo - Move the internal MBB iterator but do not update register states. + /// Move the internal MBB iterator but do not update register states. void skipTo(MachineBasicBlock::iterator I) { if (I == MachineBasicBlock::iterator(nullptr)) Tracking = false; @@ -110,14 +108,13 @@ public: return MBBI; } - /// isRegUsed - return if a specific register is currently used. + /// Return if a specific register is currently used. bool isRegUsed(unsigned Reg, bool includeReserved = true) const; - /// getRegsAvailable - Return all available registers in the register class - /// in Mask. + /// Return all available registers in the register class in Mask. BitVector getRegsAvailable(const TargetRegisterClass *RC); - /// FindUnusedReg - Find a unused register of the specified register class. + /// Find an unused register of the specified register class. /// Return 0 if none is found. unsigned FindUnusedReg(const TargetRegisterClass *RegClass) const; @@ -144,7 +141,7 @@ public: A.push_back(I->FrameIndex); } - /// scavengeRegister - Make a register of the specific register class + /// Make a register of the specific register class /// available and do the appropriate bookkeeping. SPAdj is the stack /// adjustment due to call frame, it's passed along to eliminateFrameIndex(). /// Returns the scavenged register. @@ -154,11 +151,10 @@ public: return scavengeRegister(RegClass, MBBI, SPAdj); } - /// setRegUsed - Tell the scavenger a register is used. - /// + /// Tell the scavenger a register is used. void setRegUsed(unsigned Reg); private: - /// isReserved - Returns true if a register is reserved. It is never "unused". + /// Returns true if a register is reserved. It is never "unused". bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); } /// setUsed / setUnused - Mark the state of one or a number of register units. @@ -177,9 +173,8 @@ private: /// Add all Reg Units that Reg contains to BV. void addRegUnits(BitVector &BV, unsigned Reg); - /// findSurvivorReg - Return the candidate register that is unused for the - /// longest after StartMI. UseMI is set to the instruction where the search - /// stopped. + /// Return the candidate register that is unused for the longest after + /// StartMI. UseMI is set to the instruction where the search stopped. /// /// No more than InstrLimit instructions are inspected. unsigned findSurvivorReg(MachineBasicBlock::iterator StartMI, diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h index 114fe7c35b827..0097e0472e5c5 100644 --- a/include/llvm/CodeGen/ResourcePriorityQueue.h +++ b/include/llvm/CodeGen/ResourcePriorityQueue.h @@ -64,7 +64,7 @@ namespace llvm { /// ResourcesModel - Represents VLIW state. /// Not limited to VLIW targets per say, but assumes /// definition of DFA by a target. - DFAPacketizer *ResourcesModel; + std::unique_ptr<DFAPacketizer> ResourcesModel; /// Resource model - packet/bundle model. Purely /// internal at the time. @@ -77,10 +77,6 @@ namespace llvm { public: ResourcePriorityQueue(SelectionDAGISel *IS); - ~ResourcePriorityQueue() { - delete ResourcesModel; - } - bool isBottomUp() const override { return false; } void initNodes(std::vector<SUnit> &sunits) override; @@ -124,8 +120,6 @@ namespace llvm { void remove(SUnit *SU) override; - void dump(ScheduleDAG* DAG) const override; - /// scheduledNode - Main resource tracking point. void scheduledNode(SUnit *Node) override; bool isResourceAvailable(SUnit *SU); diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h index 64c9c4729e922..2be5de640e292 100644 --- a/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/include/llvm/CodeGen/RuntimeLibcalls.h @@ -425,6 +425,10 @@ namespace RTLIB { /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or /// UNKNOWN_LIBCALL if there is none. Libcall getUINTTOFP(EVT OpVT, EVT RetVT); + + /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or + /// UNKNOWN_LIBCALL if there is none. + Libcall getATOMIC(unsigned Opc, MVT VT); } } diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 80aee8c628809..839131416560f 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -628,12 +628,6 @@ namespace llvm { } bool operator!=(const SUnitIterator& x) const { return !operator==(x); } - const SUnitIterator &operator=(const SUnitIterator &I) { - assert(I.Node==Node && "Cannot assign iterators to two different nodes!"); - Operand = I.Operand; - return *this; - } - pointer operator*() const { return Node->Preds[Operand].getSUnit(); } diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index 00dd8f9a633e2..1196783e820b8 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -158,7 +158,7 @@ namespace llvm { bool RemoveKillFlags = false, LiveIntervals *LIS = nullptr); - virtual ~ScheduleDAGInstrs() {} + ~ScheduleDAGInstrs() override {} bool isPostRA() const { return IsPostRA; } diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index ee4bc08495ec9..89f9005b8b211 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -41,14 +41,16 @@ class TargetSelectionDAGInfo; class SDVTListNode : public FoldingSetNode { friend struct FoldingSetTrait<SDVTListNode>; - /// FastID - A reference to an Interned FoldingSetNodeID for this node. + /// A reference to an Interned FoldingSetNodeID for this node. /// The Allocator in SelectionDAG holds the data. /// SDVTList contains all types which are frequently accessed in SelectionDAG. - /// The size of this list is not expected big so it won't introduce memory penalty. + /// The size of this list is not expected to be big so it won't introduce + /// a memory penalty. FoldingSetNodeIDRef FastID; const EVT *VTs; unsigned int NumVTs; - /// The hash value for SDVTList is fixed so cache it to avoid hash calculation + /// The hash value for SDVTList is fixed, so cache it to avoid + /// hash calculation. unsigned HashValue; public: SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : @@ -61,8 +63,8 @@ public: } }; -// Specialize FoldingSetTrait for SDVTListNode -// To avoid computing temp FoldingSetNodeID and hash value. +/// Specialize FoldingSetTrait for SDVTListNode +/// to avoid computing temp FoldingSetNodeID and hash value. template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> { static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { ID = X.FastID; @@ -98,7 +100,7 @@ private: static void createNode(const SDNode &); }; -/// SDDbgInfo - Keeps track of dbg_value information through SDISel. We do +/// Keeps track of dbg_value information through SDISel. We do /// not build SDNodes for these so as not to perturb the generated code; /// instead the info is kept off to the side in this structure. Each SDNode may /// have one or more associated dbg_value entries. This information is kept in @@ -110,13 +112,14 @@ private: /// motion, and debug info for them is potentially useful even if the parameter /// is unused. Right now only byval parameters are handled separately. class SDDbgInfo { + BumpPtrAllocator Alloc; SmallVector<SDDbgValue*, 32> DbgValues; SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; typedef DenseMap<const SDNode*, SmallVector<SDDbgValue*, 2> > DbgValMapType; DbgValMapType DbgValMap; - void operator=(const SDDbgInfo&) LLVM_DELETED_FUNCTION; - SDDbgInfo(const SDDbgInfo&) LLVM_DELETED_FUNCTION; + void operator=(const SDDbgInfo&) = delete; + SDDbgInfo(const SDDbgInfo&) = delete; public: SDDbgInfo() {} @@ -136,8 +139,11 @@ public: DbgValMap.clear(); DbgValues.clear(); ByvalParmDbgValues.clear(); + Alloc.Reset(); } + BumpPtrAllocator &getAlloc() { return Alloc; } + bool empty() const { return DbgValues.empty() && ByvalParmDbgValues.empty(); } @@ -159,10 +165,10 @@ public: class SelectionDAG; void checkForCycles(const SelectionDAG *DAG, bool force = false); -/// SelectionDAG class - This is used to represent a portion of an LLVM function -/// in a low-level Data Dependence DAG representation suitable for instruction -/// selection. This DAG is constructed as the first step of instruction -/// selection in order to allow implementation of machine specific optimizations +/// This is used to represent a portion of an LLVM function in a low-level +/// Data Dependence DAG representation suitable for instruction selection. +/// This DAG is constructed as the first step of instruction selection in order +/// to allow implementation of machine specific optimizations /// and code simplifications. /// /// The representation used by the SelectionDAG is a target-independent @@ -178,40 +184,39 @@ class SelectionDAG { LLVMContext *Context; CodeGenOpt::Level OptLevel; - /// EntryNode - The starting token. + /// The starting token. SDNode EntryNode; - /// Root - The root of the entire DAG. + /// The root of the entire DAG. SDValue Root; - /// AllNodes - A linked list of nodes in the current DAG. + /// A linked list of nodes in the current DAG. ilist<SDNode> AllNodes; - /// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use + /// The AllocatorType for allocating SDNodes. We use /// pool allocation with recycling. typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode), AlignOf<MostAlignedSDNode>::Alignment> NodeAllocatorType; - /// NodeAllocator - Pool allocation for nodes. + /// Pool allocation for nodes. NodeAllocatorType NodeAllocator; - /// CSEMap - This structure is used to memoize nodes, automatically performing + /// This structure is used to memoize nodes, automatically performing /// CSE with existing nodes when a duplicate is requested. FoldingSet<SDNode> CSEMap; - /// OperandAllocator - Pool allocation for machine-opcode SDNode operands. + /// Pool allocation for machine-opcode SDNode operands. BumpPtrAllocator OperandAllocator; - /// Allocator - Pool allocation for misc. objects that are created once per - /// SelectionDAG. + /// Pool allocation for misc. objects that are created once per SelectionDAG. BumpPtrAllocator Allocator; - /// DbgInfo - Tracks dbg_value information through SDISel. + /// Tracks dbg_value information through SDISel. SDDbgInfo *DbgInfo; public: - /// DAGUpdateListener - Clients of various APIs that cause global effects on + /// Clients of various APIs that cause global effects on /// the DAG can optionally implement this interface. This allows the clients /// to handle the various sorts of updates that happen. /// @@ -232,15 +237,15 @@ public: DAG.UpdateListeners = Next; } - /// NodeDeleted - The node N that was deleted and, if E is not null, an + /// The node N that was deleted and, if E is not null, an /// equivalent node E that replaced it. virtual void NodeDeleted(SDNode *N, SDNode *E); - /// NodeUpdated - The node N that was updated. + /// The node N that was updated. virtual void NodeUpdated(SDNode *N); }; - /// NewNodesMustHaveLegalTypes - When true, additional steps are taken to + /// When true, additional steps are taken to /// ensure that getConstant() and similar functions return DAG nodes that /// have legal types. This is important after type legalization since /// any illegally typed nodes generated after this point will not experience @@ -251,32 +256,28 @@ private: /// DAGUpdateListener is a friend so it can manipulate the listener stack. friend struct DAGUpdateListener; - /// UpdateListeners - Linked list of registered DAGUpdateListener instances. + /// Linked list of registered DAGUpdateListener instances. /// This stack is maintained by DAGUpdateListener RAII. DAGUpdateListener *UpdateListeners; - /// setGraphColorHelper - Implementation of setSubgraphColor. + /// Implementation of setSubgraphColor. /// Return whether we had to truncate the search. - /// bool setSubgraphColorHelper(SDNode *N, const char *Color, DenseSet<SDNode *> &visited, int level, bool &printed); - void operator=(const SelectionDAG&) LLVM_DELETED_FUNCTION; - SelectionDAG(const SelectionDAG&) LLVM_DELETED_FUNCTION; + void operator=(const SelectionDAG&) = delete; + SelectionDAG(const SelectionDAG&) = delete; public: explicit SelectionDAG(const TargetMachine &TM, llvm::CodeGenOpt::Level); ~SelectionDAG(); - /// init - Prepare this SelectionDAG to process code in the given - /// MachineFunction. - /// + /// Prepare this SelectionDAG to process code in the given MachineFunction. void init(MachineFunction &mf); - /// clear - Clear state and free memory necessary to make this + /// Clear state and free memory necessary to make this /// SelectionDAG ready to process a new block. - /// void clear(); MachineFunction &getMachineFunction() const { return *MF; } @@ -286,8 +287,7 @@ public: const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return *TSI; } LLVMContext *getContext() const {return Context; } - /// viewGraph - Pop up a GraphViz/gv window with the DAG rendered using 'dot'. - /// + /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. void viewGraph(const std::string &Title); void viewGraph(); @@ -295,24 +295,21 @@ public: std::map<const SDNode *, std::string> NodeGraphAttrs; #endif - /// clearGraphAttrs - Clear all previously defined node graph attributes. + /// Clear all previously defined node graph attributes. /// Intended to be used from a debugging tool (eg. gdb). void clearGraphAttrs(); - /// setGraphAttrs - Set graph attributes for a node. (eg. "color=red".) - /// + /// Set graph attributes for a node. (eg. "color=red".) void setGraphAttrs(const SDNode *N, const char *Attrs); - /// getGraphAttrs - Get graph attributes for a node. (eg. "color=red".) + /// Get graph attributes for a node. (eg. "color=red".) /// Used from getNodeAttributes. const std::string getGraphAttrs(const SDNode *N) const; - /// setGraphColor - Convenience for setting node color attribute. - /// + /// Convenience for setting node color attribute. void setGraphColor(const SDNode *N, const char *Color); - /// setGraphColor - Convenience for setting subgraph color attribute. - /// + /// Convenience for setting subgraph color attribute. void setSubgraphColor(SDNode *N, const char *Color); typedef ilist<SDNode>::const_iterator allnodes_const_iterator; @@ -325,17 +322,15 @@ public: return AllNodes.size(); } - /// getRoot - Return the root tag of the SelectionDAG. - /// + /// Return the root tag of the SelectionDAG. const SDValue &getRoot() const { return Root; } - /// getEntryNode - Return the token chain corresponding to the entry of the - /// function. + /// Return the token chain corresponding to the entry of the function. SDValue getEntryNode() const { return SDValue(const_cast<SDNode *>(&EntryNode), 0); } - /// setRoot - Set the current root tag of the SelectionDAG. + /// Set the current root tag of the SelectionDAG. /// const SDValue &setRoot(SDValue N) { assert((!N.getNode() || N.getValueType() == MVT::Other) && @@ -348,22 +343,22 @@ public: return Root; } - /// Combine - This iterates over the nodes in the SelectionDAG, folding + /// This iterates over the nodes in the SelectionDAG, folding /// certain types of nodes together, or eliminating superfluous nodes. The /// Level argument controls whether Combine is allowed to produce nodes and /// types that are illegal on the target. void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel); - /// LegalizeTypes - This transforms the SelectionDAG into a SelectionDAG that - /// only uses types natively supported by the target. Returns "true" if it - /// made any changes. + /// This transforms the SelectionDAG into a SelectionDAG that + /// only uses types natively supported by the target. + /// Returns "true" if it made any changes. /// /// Note that this is an involved process that may invalidate pointers into /// the graph. bool LegalizeTypes(); - /// Legalize - This transforms the SelectionDAG into a SelectionDAG that is + /// This transforms the SelectionDAG into a SelectionDAG that is /// compatible with the target instruction selector, as indicated by the /// TargetLowering object. /// @@ -392,7 +387,7 @@ public: /// UpdatedNodes with any new nodes replacing those originally in the DAG. bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes); - /// LegalizeVectors - This transforms the SelectionDAG into a SelectionDAG + /// This transforms the SelectionDAG into a SelectionDAG /// that only uses vector math operations supported by the target. This is /// necessary as a separate step from Legalize because unrolling a vector /// operation can introduce illegal types, which requires running @@ -405,16 +400,14 @@ public: /// the graph. bool LegalizeVectors(); - /// RemoveDeadNodes - This method deletes all unreachable nodes in the - /// SelectionDAG. + /// This method deletes all unreachable nodes in the SelectionDAG. void RemoveDeadNodes(); - /// DeleteNode - Remove the specified node from the system. This node must + /// Remove the specified node from the system. This node must /// have no referrers. void DeleteNode(SDNode *N); - /// getVTList - Return an SDVTList that represents the list of values - /// specified. + /// Return an SDVTList that represents the list of values specified. SDVTList getVTList(EVT VT); SDVTList getVTList(EVT VT1, EVT VT2); SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3); @@ -424,36 +417,40 @@ public: //===--------------------------------------------------------------------===// // Node creation methods. // - SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false, + SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget = false, bool isOpaque = false); - SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false, + SDValue getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isTarget = false, bool isOpaque = false); - SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false, - bool isOpaque = false); - SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false); - SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) { - return getConstant(Val, VT, true, isOpaque); + SDValue getConstant(const ConstantInt &Val, SDLoc DL, EVT VT, + bool isTarget = false, bool isOpaque = false); + SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget = false); + SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, + bool isOpaque = false) { + return getConstant(Val, DL, VT, true, isOpaque); } - SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) { - return getConstant(Val, VT, true, isOpaque); + SDValue getTargetConstant(const APInt &Val, SDLoc DL, EVT VT, + bool isOpaque = false) { + return getConstant(Val, DL, VT, true, isOpaque); } - SDValue getTargetConstant(const ConstantInt &Val, EVT VT, + SDValue getTargetConstant(const ConstantInt &Val, SDLoc DL, EVT VT, bool isOpaque = false) { - return getConstant(Val, VT, true, isOpaque); + return getConstant(Val, DL, VT, true, isOpaque); } // The forms below that take a double should only be used for simple // constants that can be exactly represented in VT. No checks are made. - SDValue getConstantFP(double Val, EVT VT, bool isTarget = false); - SDValue getConstantFP(const APFloat& Val, EVT VT, bool isTarget = false); - SDValue getConstantFP(const ConstantFP &CF, EVT VT, bool isTarget = false); - SDValue getTargetConstantFP(double Val, EVT VT) { - return getConstantFP(Val, VT, true); + SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget = false); + SDValue getConstantFP(const APFloat& Val, SDLoc DL, EVT VT, + bool isTarget = false); + SDValue getConstantFP(const ConstantFP &CF, SDLoc DL, EVT VT, + bool isTarget = false); + SDValue getTargetConstantFP(double Val, SDLoc DL, EVT VT) { + return getConstantFP(Val, DL, VT, true); } - SDValue getTargetConstantFP(const APFloat& Val, EVT VT) { - return getConstantFP(Val, VT, true); + SDValue getTargetConstantFP(const APFloat& Val, SDLoc DL, EVT VT) { + return getConstantFP(Val, DL, VT, true); } - SDValue getTargetConstantFP(const ConstantFP &Val, EVT VT) { - return getConstantFP(Val, VT, true); + SDValue getTargetConstantFP(const ConstantFP &Val, SDLoc DL, EVT VT) { + return getConstantFP(Val, DL, VT, true); } SDValue getGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT, int64_t offset = 0, bool isTargetGA = false, @@ -561,10 +558,9 @@ public: SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code); - /// getVectorShuffle - Return an ISD::VECTOR_SHUFFLE node. The number of - /// elements in VT, which must be a vector type, must match the number of - /// mask elements NumElts. A integer mask element equal to -1 is treated as - /// undefined. + /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT, + /// which must be a vector type, must match the number of mask elements + /// NumElts. An integer mask element equal to -1 is treated as undefined. SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts); SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, @@ -580,63 +576,62 @@ public: /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3> SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV); - /// getAnyExtOrTrunc - Convert Op, which must be of integer type, to the + /// Convert Op, which must be of integer type, to the /// integer type VT, by either any-extending or truncating it. SDValue getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT); - /// getSExtOrTrunc - Convert Op, which must be of integer type, to the + /// Convert Op, which must be of integer type, to the /// integer type VT, by either sign-extending or truncating it. SDValue getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT); - /// getZExtOrTrunc - Convert Op, which must be of integer type, to the + /// Convert Op, which must be of integer type, to the /// integer type VT, by either zero-extending or truncating it. SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT); - /// getZeroExtendInReg - Return the expression required to zero extend the Op + /// Return the expression required to zero extend the Op /// value assuming it was the smaller SrcTy value. SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy); - /// getAnyExtendVectorInReg - Return an operation which will any-extend the - /// low lanes of the operand into the specified vector type. For example, + /// Return an operation which will any-extend the low lanes of the operand + /// into the specified vector type. For example, /// this can convert a v16i8 into a v4i32 by any-extending the low four /// lanes of the operand from i8 to i32. SDValue getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT); - /// getSignExtendVectorInReg - Return an operation which will sign extend the - /// low lanes of the operand into the specified vector type. For example, + /// Return an operation which will sign extend the low lanes of the operand + /// into the specified vector type. For example, /// this can convert a v16i8 into a v4i32 by sign extending the low four /// lanes of the operand from i8 to i32. SDValue getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT); - /// getZeroExtendVectorInReg - Return an operation which will zero extend the - /// low lanes of the operand into the specified vector type. For example, + /// Return an operation which will zero extend the low lanes of the operand + /// into the specified vector type. For example, /// this can convert a v16i8 into a v4i32 by zero extending the low four /// lanes of the operand from i8 to i32. SDValue getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT); - /// getBoolExtOrTrunc - Convert Op, which must be of integer type, to the - /// integer type VT, by using an extension appropriate for the target's + /// Convert Op, which must be of integer type, to the integer type VT, + /// by using an extension appropriate for the target's /// BooleanContent for type OpVT or truncating it. SDValue getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT, EVT OpVT); - /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). + /// Create a bitwise NOT operation as (XOR Val, -1). SDValue getNOT(SDLoc DL, SDValue Val, EVT VT); /// \brief Create a logical NOT operation as (XOR Val, BooleanOne). SDValue getLogicalNOT(SDLoc DL, SDValue Val, EVT VT); - /// getCALLSEQ_START - Return a new CALLSEQ_START node, which always must have - /// a glue result (to ensure it's not CSE'd). CALLSEQ_START does not have a - /// useful SDLoc. + /// Return a new CALLSEQ_START node, which always must have a glue result + /// (to ensure it's not CSE'd). CALLSEQ_START does not have a useful SDLoc. SDValue getCALLSEQ_START(SDValue Chain, SDValue Op, SDLoc DL) { SDVTList VTs = getVTList(MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, Op }; return getNode(ISD::CALLSEQ_START, DL, VTs, Ops); } - /// getCALLSEQ_END - Return a new CALLSEQ_END node, which always must have a - /// glue result (to ensure it's not CSE'd). CALLSEQ_END does not have - /// a useful SDLoc. + /// Return a new CALLSEQ_END node, which always must have a + /// glue result (to ensure it's not CSE'd). + /// CALLSEQ_END does not have a useful SDLoc. SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, SDLoc DL) { SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue); @@ -649,19 +644,28 @@ public: return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops); } - /// getUNDEF - Return an UNDEF node. UNDEF does not have a useful SDLoc. + /// Return an UNDEF node. UNDEF does not have a useful SDLoc. SDValue getUNDEF(EVT VT) { return getNode(ISD::UNDEF, SDLoc(), VT); } - /// getGLOBAL_OFFSET_TABLE - Return a GLOBAL_OFFSET_TABLE node. This does - /// not have a useful SDLoc. + /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc. SDValue getGLOBAL_OFFSET_TABLE(EVT VT) { return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); } - /// getNode - Gets or creates the specified node. + /// Gets or creates the specified node. /// + SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, + ArrayRef<SDUse> Ops); + SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, + ArrayRef<SDValue> Ops); + SDValue getNode(unsigned Opcode, SDLoc DL, ArrayRef<EVT> ResultTys, + ArrayRef<SDValue> Ops); + SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, + ArrayRef<SDValue> Ops); + + // Specialize based on number of operands. SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, @@ -672,47 +676,40 @@ public: SDValue N3, SDValue N4); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, SDValue N3, SDValue N4, SDValue N5); - SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef<SDUse> Ops); - SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, - ArrayRef<SDValue> Ops); - SDValue getNode(unsigned Opcode, SDLoc DL, - ArrayRef<EVT> ResultTys, - ArrayRef<SDValue> Ops); - SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - ArrayRef<SDValue> Ops); + + // Specialize again based on number of operands for nodes with a VTList + // rather than a single VT. SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs); SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N); - SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2); - SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, SDValue N3); - SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, SDValue N3, SDValue N4); - SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, SDValue N3, SDValue N4, - SDValue N5); - - /// getStackArgumentTokenFactor - Compute a TokenFactor to force all - /// the incoming stack arguments to be loaded from the stack. This is - /// used in tail call lowering to protect stack arguments from being - /// clobbered. + SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, + SDValue N2); + SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, + SDValue N2, SDValue N3); + SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, + SDValue N2, SDValue N3, SDValue N4); + SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, + SDValue N2, SDValue N3, SDValue N4, SDValue N5); + + /// Compute a TokenFactor to force all the incoming stack arguments to be + /// loaded from the stack. This is used in tail call lowering to protect + /// stack arguments from being clobbered. SDValue getStackArgumentTokenFactor(SDValue Chain); SDValue getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, - MachinePointerInfo DstPtrInfo, + bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo); SDValue getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, - SDValue Size, unsigned Align, bool isVol, + SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo); SDValue getMemset(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, - SDValue Size, unsigned Align, bool isVol, + SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo); - /// getSetCC - Helper function to make it easier to build SetCC's if you just + /// Helper function to make it easier to build SetCC's if you just /// have an ISD::CondCode instead of an SDValue. /// SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, @@ -726,8 +723,8 @@ public: return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); } - // getSelect - Helper function to make it easier to build Select's if you just - // have operands and don't want to check for vector. + /// Helper function to make it easier to build Select's if you just + /// have operands and don't want to check for vector. SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS) { assert(LHS.getValueType() == RHS.getValueType() && @@ -738,7 +735,7 @@ public: Cond, LHS, RHS); } - /// getSelectCC - Helper function to make it easier to build SelectCC's if you + /// Helper function to make it easier to build SelectCC's if you /// just have an ISD::CondCode instead of an SDValue. /// SDValue getSelectCC(SDLoc DL, SDValue LHS, SDValue RHS, @@ -747,12 +744,12 @@ public: LHS, RHS, True, False, getCondCode(Cond)); } - /// getVAArg - VAArg produces a result and token chain, and takes a pointer + /// VAArg produces a result and token chain, and takes a pointer /// and a source value as input. SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align); - /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two + /// Gets a node for an atomic cmpxchg op. There are two /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, /// a success flag (initially i1), and a chain. @@ -769,7 +766,7 @@ public: AtomicOrdering FailureOrdering, SynchronizationScope SynchScope); - /// getAtomic - Gets a node for an atomic op, produces result (if relevant) + /// Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, @@ -780,15 +777,15 @@ public: AtomicOrdering Ordering, SynchronizationScope SynchScope); - /// getAtomic - Gets a node for an atomic op, produces result and chain and + /// Gets a node for an atomic op, produces result and chain and /// takes 1 operand. SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, AtomicOrdering Ordering, SynchronizationScope SynchScope); - /// getAtomic - Gets a node for an atomic op, produces result and chain and - /// takes N operands. + /// Gets a node for an atomic op, produces result and chain and takes N + /// operands. SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, AtomicOrdering SuccessOrdering, @@ -798,7 +795,7 @@ public: ArrayRef<SDValue> Ops, MachineMemOperand *MMO, AtomicOrdering Ordering, SynchronizationScope SynchScope); - /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a + /// Creates a MemIntrinsicNode that may produce a /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not /// less than FIRST_TARGET_MEMORY_OPCODE. @@ -813,10 +810,10 @@ public: ArrayRef<SDValue> Ops, EVT MemVT, MachineMemOperand *MMO); - /// getMergeValues - Create a MERGE_VALUES node from the given operands. + /// Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl); - /// getLoad - Loads are not normal binary operators: their result type is not + /// Loads are not normal binary operators: their result type is not /// determined by their operands, and they produce a value AND a token chain. /// SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, @@ -848,8 +845,7 @@ public: SDValue Chain, SDValue Ptr, SDValue Offset, EVT MemVT, MachineMemOperand *MMO); - /// getStore - Helper function to build ISD::STORE nodes. - /// + /// Helper function to build ISD::STORE nodes. SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, @@ -872,21 +868,25 @@ public: SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTrunc); - /// getSrcValue - Construct a node to track a Value* through the backend. + SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, + ArrayRef<SDValue> Ops, MachineMemOperand *MMO); + SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, + ArrayRef<SDValue> Ops, MachineMemOperand *MMO); + /// Construct a node to track a Value* through the backend. SDValue getSrcValue(const Value *v); - /// getMDNode - Return an MDNodeSDNode which holds an MDNode. + /// Return an MDNodeSDNode which holds an MDNode. SDValue getMDNode(const MDNode *MD); - /// getAddrSpaceCast - Return an AddrSpaceCastSDNode. + /// Return an AddrSpaceCastSDNode. SDValue getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS); - /// getShiftAmountOperand - Return the specified value casted to + /// Return the specified value casted to /// the target's desired shift amount type. SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); - /// UpdateNodeOperands - *Mutate* the specified node in-place to have the + /// *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that /// already exists. If the resultant node does not exist in the DAG, the @@ -902,7 +902,7 @@ public: SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); - /// SelectNodeTo - These are used for target selectors to *mutate* the + /// These are used for target selectors to *mutate* the /// specified node to have the specified return type, Target opcode, and /// operands. Note that target opcodes are stored as /// ~TargetOpcode in the node opcode field. The resultant node is returned. @@ -932,12 +932,12 @@ public: SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, SDVTList VTs, ArrayRef<SDValue> Ops); - /// MorphNodeTo - This *mutates* the specified node to have the specified + /// This *mutates* the specified node to have the specified /// return type, opcode, and operands. SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef<SDValue> Ops); - /// getMachineNode - These are used for target selectors to create a new node + /// These are used for target selectors to create a new node /// with specified return type(s), MachineInstr opcode, and operands. /// /// Note that getMachineNode returns the resultant node. If there is already @@ -976,25 +976,20 @@ public: MachineSDNode *getMachineNode(unsigned Opcode, SDLoc dl, SDVTList VTs, ArrayRef<SDValue> Ops); - /// getTargetExtractSubreg - A convenience function for creating - /// TargetInstrInfo::EXTRACT_SUBREG nodes. + /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes. SDValue getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, SDValue Operand); - /// getTargetInsertSubreg - A convenience function for creating - /// TargetInstrInfo::INSERT_SUBREG nodes. + /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes. SDValue getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, SDValue Operand, SDValue Subreg); - /// getNodeIfExists - Get the specified node if it's already available, or - /// else return NULL. + /// Get the specified node if it's already available, or else return NULL. SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops, bool nuw = false, bool nsw = false, bool exact = false); - /// getDbgValue - Creates a SDDbgValue node. - /// - /// SDNode + /// Creates a SDDbgValue node. SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool IsIndirect, uint64_t Off, DebugLoc DL, unsigned O); @@ -1007,16 +1002,16 @@ public: SDDbgValue *getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t Off, DebugLoc DL, unsigned O); - /// RemoveDeadNode - Remove the specified node from the system. If any of its + /// Remove the specified node from the system. If any of its /// operands then becomes dead, remove them as well. Inform UpdateListener /// for each node deleted. void RemoveDeadNode(SDNode *N); - /// RemoveDeadNodes - This method deletes the unreachable nodes in the + /// This method deletes the unreachable nodes in the /// given list, and any nodes that become unreachable as a result. void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes); - /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. + /// Modify anything using 'From' to use 'To' instead. /// This can cause recursive merging of nodes in the DAG. Use the first /// version if 'From' is known to have a single result, use the second /// if you have two nodes with identical results (or if 'To' has a superset @@ -1035,30 +1030,29 @@ public: void ReplaceAllUsesWith(SDNode *From, SDNode *To); void ReplaceAllUsesWith(SDNode *From, const SDValue *To); - /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving + /// Replace any uses of From with To, leaving /// uses of other values produced by From.Val alone. void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); - /// ReplaceAllUsesOfValuesWith - Like ReplaceAllUsesOfValueWith, but - /// for multiple values at once. This correctly handles the case where + /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. + /// This correctly handles the case where /// there is an overlap between the From values and the To values. void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num); - /// AssignTopologicalOrder - Topological-sort the AllNodes list and a + /// Topological-sort the AllNodes list and a /// assign a unique node id for each node in the DAG based on their /// topological order. Returns the number of nodes. unsigned AssignTopologicalOrder(); - /// RepositionNode - Move node N in the AllNodes list to be immediately + /// Move node N in the AllNodes list to be immediately /// before the given iterator Position. This may be used to update the /// topological ordering when the list of nodes is modified. void RepositionNode(allnodes_iterator Position, SDNode *N) { AllNodes.insert(Position, AllNodes.remove(N)); } - /// isCommutativeBinOp - Returns true if the opcode is a commutative binary - /// operation. + /// Returns true if the opcode is a commutative binary operation. static bool isCommutativeBinOp(unsigned Opcode) { // FIXME: This should get its info from the td file, so that we can include // target info. @@ -1099,19 +1093,19 @@ public: } } - /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the + /// Add a dbg_value SDNode. If SD is non-null that means the /// value is produced by SD. void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter); - /// GetDbgValues - Get the debug values which reference the given SDNode. + /// Get the debug values which reference the given SDNode. ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) { return DbgInfo->getSDDbgValues(SD); } - /// TransferDbgValues - Transfer SDDbgValues. + /// Transfer SDDbgValues. void TransferDbgValues(SDValue From, SDValue To); - /// hasDebugValues - Return true if there are any SDDbgValue nodes associated + /// Return true if there are any SDDbgValue nodes associated /// with this SelectionDAG. bool hasDebugValues() const { return !DbgInfo->empty(); } @@ -1126,28 +1120,31 @@ public: void dump() const; - /// CreateStackTemporary - Create a stack temporary, suitable for holding the + /// Create a stack temporary, suitable for holding the /// specified value type. If minAlign is specified, the slot size will have /// at least that alignment. SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); - /// CreateStackTemporary - Create a stack temporary suitable for holding + /// Create a stack temporary suitable for holding /// either of the specified value types. SDValue CreateStackTemporary(EVT VT1, EVT VT2); - /// FoldConstantArithmetic - - SDValue FoldConstantArithmetic(unsigned Opcode, EVT VT, + SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT, SDNode *Cst1, SDNode *Cst2); - /// FoldSetCC - Constant fold a setcc to true or false. + SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT, + const ConstantSDNode *Cst1, + const ConstantSDNode *Cst2); + + /// Constant fold a setcc to true or false. SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, SDLoc dl); - /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We - /// use this predicate to simplify operations downstream. + /// Return true if the sign bit of Op is known to be zero. + /// We use this predicate to simplify operations downstream. bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; - /// MaskedValueIsZero - Return true if 'Op & Mask' is known to be zero. We + /// Return true if 'Op & Mask' is known to be zero. We /// use this predicate to simplify operations downstream. Op and Mask are /// known to be the same type. bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth = 0) @@ -1160,7 +1157,7 @@ public: void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth = 0) const; - /// ComputeNumSignBits - Return the number of times the sign bit of the + /// Return the number of times the sign bit of the /// register is replicated into the other bits. We know that at least 1 bit /// is always equal to the sign bit (itself), but other cases can give us /// information. For example, immediately after an "SRA X, 2", we know that @@ -1169,26 +1166,26 @@ public: /// class to allow target nodes to be understood. unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const; - /// isBaseWithConstantOffset - Return true if the specified operand is an + /// Return true if the specified operand is an /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an /// ISD::OR with a ConstantSDNode that is guaranteed to have the same /// semantics as an ADD. This handles the equivalence: /// X|Cst == X+Cst iff X&Cst = 0. bool isBaseWithConstantOffset(SDValue Op) const; - /// isKnownNeverNan - Test whether the given SDValue is known to never be NaN. + /// Test whether the given SDValue is known to never be NaN. bool isKnownNeverNaN(SDValue Op) const; - /// isKnownNeverZero - Test whether the given SDValue is known to never be + /// Test whether the given SDValue is known to never be /// positive or negative Zero. bool isKnownNeverZero(SDValue Op) const; - /// isEqualTo - Test whether two SDValues are known to compare equal. This + /// Test whether two SDValues are known to compare equal. This /// is true if they are the same value, or if one is negative zero and the /// other positive zero. bool isEqualTo(SDValue A, SDValue B) const; - /// UnrollVectorOp - Utility function used by legalize and lowering to + /// Utility function used by legalize and lowering to /// "unroll" a vector operation by splitting out the scalars and operating /// on each element individually. If the ResNE is 0, fully unroll the vector /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. @@ -1196,43 +1193,40 @@ public: /// vector op and fill the end of the resulting vector with UNDEFS. SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); - /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a - /// location that is 'Dist' units away from the location that the 'Base' load - /// is loading from. + /// Return true if LD is loading 'Bytes' bytes from a location that is 'Dist' + /// units away from the location that the 'Base' load is loading from. bool isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const; - /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if + /// Infer alignment of a load / store address. Return 0 if /// it cannot be inferred. unsigned InferPtrAlignment(SDValue Ptr) const; - /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type + /// Compute the VTs needed for the low/hi parts of a type /// which is split (or expanded) into two not necessarily identical pieces. std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const; - /// SplitVector - Split the vector with EXTRACT_SUBVECTOR using the provides + /// Split the vector with EXTRACT_SUBVECTOR using the provides /// VTs and return the low/high part. std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT); - /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the - /// low/high part. + /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part. std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { EVT LoVT, HiVT; std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); return SplitVector(N, DL, LoVT, HiVT); } - /// SplitVectorOperand - Split the node's operand with EXTRACT_SUBVECTOR and + /// Split the node's operand with EXTRACT_SUBVECTOR and /// return the low/high part. std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) { return SplitVector(N->getOperand(OpNo), SDLoc(N)); } - /// ExtractVectorElements - Append the extracted elements from Start to Count - /// out of the vector Op in Args. If Count is 0, all of the elements will be - /// extracted. + /// Append the extracted elements from Start to Count out of the vector Op + /// in Args. If Count is 0, all of the elements will be extracted. void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, unsigned Start = 0, unsigned Count = 0); @@ -1258,10 +1252,22 @@ private: SDValue N1, SDValue N2, bool nuw, bool nsw, bool exact); - /// VTList - List of non-single value types. + /// Look up the node specified by ID in CSEMap. If it exists, return it. If + /// not, return the insertion token that will make insertion faster. This + /// overload is for nodes other than Constant or ConstantFP, use the other one + /// for those. + SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos); + + /// Look up the node specified by ID in CSEMap. If it exists, return it. If + /// not, return the insertion token that will make insertion faster. Performs + /// additional processing for constant nodes. + SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, DebugLoc DL, + void *&InsertPos); + + /// List of non-single value types. FoldingSet<SDVTListNode> VTListMap; - /// CondCodeNodes - Maps to auto-CSE operations. + /// Maps to auto-CSE operations. std::vector<CondCodeSDNode*> CondCodeNodes; std::vector<SDNode*> ValueTypeNodes; diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index d53e66da5a71c..a011e4c338c4a 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -58,7 +58,7 @@ public: explicit SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL = CodeGenOpt::Default); - virtual ~SelectionDAGISel(); + ~SelectionDAGISel() override; const TargetLowering *getTargetLowering() const { return TLI; } @@ -80,12 +80,12 @@ public: virtual SDNode *Select(SDNode *N) = 0; /// SelectInlineAsmMemoryOperand - Select the specified address as a target - /// addressing mode, according to the specified constraint code. If this does + /// addressing mode, according to the specified constraint. If this does /// not match or is not implemented, return true. The resultant operands /// (which will appear in the machine instruction) should be added to the /// OutOps vector. virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, - char ConstraintCode, + unsigned ConstraintID, std::vector<SDValue> &OutOps) { return true; } @@ -199,7 +199,7 @@ protected: /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated /// by tblgen. Others should not call it. - void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops); + void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SDLoc DL); public: @@ -260,7 +260,10 @@ private: SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTs, ArrayRef<SDValue> Ops, unsigned EmitNodeInfo); - void PrepareEHLandingPad(); + /// Prepares the landing pad to take incoming values or do other EH + /// personality specific tasks. Returns true if the block should be + /// instruction selected, false if no code should be emitted for it. + bool PrepareEHLandingPad(); /// \brief Perform instruction selection on all basic blocks in the function. void SelectAllBasicBlocks(const Function &Fn); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 933ec7fff4839..daa19430192a5 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -50,28 +50,10 @@ template <typename T> struct DenseMapInfo; template <typename T> struct simplify_type; template <typename T> struct ilist_traits; -/// isBinOpWithFlags - Returns true if the opcode is a binary operation -/// with flags. -static bool isBinOpWithFlags(unsigned Opcode) { - switch (Opcode) { - case ISD::SDIV: - case ISD::UDIV: - case ISD::SRA: - case ISD::SRL: - case ISD::MUL: - case ISD::ADD: - case ISD::SUB: - case ISD::SHL: - return true; - default: - return false; - } -} - void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, bool force = false); -/// SDVTList - This represents a list of ValueType's that has been intern'd by +/// This represents a list of ValueType's that has been intern'd by /// a SelectionDAG. Instances of this simple value class are returned by /// SelectionDAG::getVTList(...). /// @@ -83,11 +65,11 @@ struct SDVTList { namespace ISD { /// Node predicates - /// isBuildVectorAllOnes - Return true if the specified node is a + /// Return true if the specified node is a /// BUILD_VECTOR where all of the elements are ~0 or undef. bool isBuildVectorAllOnes(const SDNode *N); - /// isBuildVectorAllZeros - Return true if the specified node is a + /// Return true if the specified node is a /// BUILD_VECTOR where all of the elements are 0 or undef. bool isBuildVectorAllZeros(const SDNode *N); @@ -95,18 +77,22 @@ namespace ISD { /// all ConstantSDNode or undef. bool isBuildVectorOfConstantSDNodes(const SDNode *N); - /// isScalarToVector - Return true if the specified node is a + /// \brief Return true if the specified node is a BUILD_VECTOR node of + /// all ConstantFPSDNode or undef. + bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); + + /// Return true if the specified node is a /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low /// element is not an undef. bool isScalarToVector(const SDNode *N); - /// allOperandsUndef - Return true if the node has at least one operand + /// Return true if the node has at least one operand /// and all operands of the specified node are ISD::UNDEF. bool allOperandsUndef(const SDNode *N); } // end llvm:ISD namespace //===----------------------------------------------------------------------===// -/// SDValue - Unlike LLVM values, Selection DAG nodes may return multiple +/// Unlike LLVM values, Selection DAG nodes may return multiple /// values as the result of a computation. Many nodes return multiple values, /// from loads (which define a token and a return value) to ADDC (which returns /// a result and a carry value), to calls (which may return an arbitrary number @@ -145,7 +131,7 @@ public: bool operator<(const SDValue &O) const { return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); } - LLVM_EXPLICIT operator bool() const { + explicit operator bool() const { return Node != nullptr; } @@ -153,11 +139,10 @@ public: return SDValue(Node, R); } - // isOperandOf - Return true if this node is an operand of N. + // Return true if this node is an operand of N. bool isOperandOf(SDNode *N) const; - /// getValueType - Return the ValueType of the referenced return value. - /// + /// Return the ValueType of the referenced return value. inline EVT getValueType() const; /// Return the simple ValueType of the referenced return value. @@ -165,8 +150,7 @@ public: return getValueType().getSimpleVT(); } - /// getValueSizeInBits - Returns the size of the value in bits. - /// + /// Returns the size of the value in bits. unsigned getValueSizeInBits() const { return getValueType().getSizeInBits(); } @@ -184,26 +168,22 @@ public: inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; inline unsigned getMachineOpcode() const; - inline const DebugLoc getDebugLoc() const; + inline const DebugLoc &getDebugLoc() const; inline void dump() const; inline void dumpr() const; - /// reachesChainWithoutSideEffects - Return true if this operand (which must - /// be a chain) reaches the specified operand without crossing any - /// side-effecting instructions. In practice, this looks through token - /// factors and non-volatile loads. In order to remain efficient, this only + /// Return true if this operand (which must be a chain) reaches the + /// specified operand without crossing any side-effecting instructions. + /// In practice, this looks through token factors and non-volatile loads. + /// In order to remain efficient, this only /// looks a couple of nodes in, it does not do an exhaustive search. bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth = 2) const; - /// use_empty - Return true if there are no nodes using value ResNo - /// of Node. - /// + /// Return true if there are no nodes using value ResNo of Node. inline bool use_empty() const; - /// hasOneUse - Return true if there is exactly one node using value - /// ResNo of Node. - /// + /// Return true if there is exactly one node using value ResNo of Node. inline bool hasOneUse() const; }; @@ -230,7 +210,7 @@ template<> struct DenseMapInfo<SDValue> { template <> struct isPodLike<SDValue> { static const bool value = true; }; -/// simplify_type specializations - Allow casting operators to work directly on +/// Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. template<> struct simplify_type<SDValue> { typedef SDNode* SimpleType; @@ -245,7 +225,7 @@ template<> struct simplify_type<const SDValue> { } }; -/// SDUse - Represents a use of a SDNode. This class holds an SDValue, +/// Represents a use of a SDNode. This class holds an SDValue, /// which records the SDNode being used and the result number, a /// pointer to the SDNode using the value, and Next and Prev pointers, /// which link together all the uses of an SDNode. @@ -259,8 +239,8 @@ class SDUse { /// this operand. SDUse **Prev, *Next; - SDUse(const SDUse &U) LLVM_DELETED_FUNCTION; - void operator=(const SDUse &U) LLVM_DELETED_FUNCTION; + SDUse(const SDUse &U) = delete; + void operator=(const SDUse &U) = delete; public: SDUse() : Val(), User(nullptr), Prev(nullptr), Next(nullptr) {} @@ -272,30 +252,30 @@ public: /// the SDValue. const SDValue &get() const { return Val; } - /// getUser - This returns the SDNode that contains this Use. + /// This returns the SDNode that contains this Use. SDNode *getUser() { return User; } - /// getNext - Get the next SDUse in the use list. + /// Get the next SDUse in the use list. SDUse *getNext() const { return Next; } - /// getNode - Convenience function for get().getNode(). + /// Convenience function for get().getNode(). SDNode *getNode() const { return Val.getNode(); } - /// getResNo - Convenience function for get().getResNo(). + /// Convenience function for get().getResNo(). unsigned getResNo() const { return Val.getResNo(); } - /// getValueType - Convenience function for get().getValueType(). + /// Convenience function for get().getValueType(). EVT getValueType() const { return Val.getValueType(); } - /// operator== - Convenience function for get().operator== + /// Convenience function for get().operator== bool operator==(const SDValue &V) const { return Val == V; } - /// operator!= - Convenience function for get().operator!= + /// Convenience function for get().operator!= bool operator!=(const SDValue &V) const { return Val != V; } - /// operator< - Convenience function for get().operator< + /// Convenience function for get().operator< bool operator<(const SDValue &V) const { return Val < V; } @@ -306,13 +286,13 @@ private: void setUser(SDNode *p) { User = p; } - /// set - Remove this use from its existing use list, assign it the + /// Remove this use from its existing use list, assign it the /// given value, and add it to the new value's node's use list. inline void set(const SDValue &V); - /// setInitial - like set, but only supports initializing a newly-allocated + /// Like set, but only supports initializing a newly-allocated /// SDUse with a non-null value. inline void setInitial(const SDValue &V); - /// setNode - like set, but only sets the Node portion of the value, + /// Like set, but only sets the Node portion of the value, /// leaving the ResNo portion unmodified. inline void setNode(SDNode *N); @@ -339,47 +319,45 @@ template<> struct simplify_type<SDUse> { }; -/// SDNode - Represents one node in the SelectionDAG. +/// Represents one node in the SelectionDAG. /// class SDNode : public FoldingSetNode, public ilist_node<SDNode> { private: - /// NodeType - The operation that this node performs. - /// + /// The operation that this node performs. int16_t NodeType; - /// OperandsNeedDelete - This is true if OperandList was new[]'d. If true, + /// This is true if OperandList was new[]'d. If true, /// then they will be delete[]'d when the node is destroyed. uint16_t OperandsNeedDelete : 1; - /// HasDebugValue - This tracks whether this node has one or more dbg_value + /// This tracks whether this node has one or more dbg_value /// nodes corresponding to it. uint16_t HasDebugValue : 1; protected: - /// SubclassData - This member is defined by this class, but is not used for + /// This member is defined by this class, but is not used for /// anything. Subclasses can use it to hold whatever state they find useful. /// This field is initialized to zero by the ctor. uint16_t SubclassData : 14; private: - /// NodeId - Unique id per SDNode in the DAG. + /// Unique id per SDNode in the DAG. int NodeId; - /// OperandList - The values that are used by this operation. - /// + /// The values that are used by this operation. SDUse *OperandList; - /// ValueList - The types of the values this node defines. SDNode's may + /// The types of the values this node defines. SDNode's may /// define multiple values simultaneously. const EVT *ValueList; - /// UseList - List of uses for this SDNode. + /// List of uses for this SDNode. SDUse *UseList; - /// NumOperands/NumValues - The number of entries in the Operand/Value list. + /// The number of entries in the Operand/Value list. unsigned short NumOperands, NumValues; - /// debugLoc - source line information. + /// Source line information. DebugLoc debugLoc; // The ordering of the SDNodes. It roughly corresponds to the ordering of the @@ -389,7 +367,7 @@ private: // this ordering. unsigned IROrder; - /// getValueTypeList - Return a pointer to the specified value type. + /// Return a pointer to the specified value type. static const EVT *getValueTypeList(EVT VT); friend class SelectionDAG; @@ -400,17 +378,17 @@ public: // Accessors // - /// getOpcode - Return the SelectionDAG opcode value for this node. For + /// Return the SelectionDAG opcode value for this node. For /// pre-isel nodes (those for which isMachineOpcode returns false), these /// are the opcode values in the ISD and <target>ISD namespaces. For /// post-isel opcodes, see getMachineOpcode. unsigned getOpcode() const { return (unsigned short)NodeType; } - /// isTargetOpcode - Test if this node has a target-specific opcode (in the + /// Test if this node has a target-specific opcode (in the /// \<target\>ISD namespace). bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } - /// isTargetMemoryOpcode - Test if this node has a target-specific + /// Test if this node has a target-specific /// memory-referencing opcode (in the \<target\>ISD namespace and /// greater than FIRST_TARGET_MEMORY_OPCODE). bool isTargetMemoryOpcode() const { @@ -427,11 +405,11 @@ public: NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); } - /// isMachineOpcode - Test if this node has a post-isel opcode, directly + /// Test if this node has a post-isel opcode, directly /// corresponding to a MachineInstr opcode. bool isMachineOpcode() const { return NodeType < 0; } - /// getMachineOpcode - This may only be called if isMachineOpcode returns + /// This may only be called if isMachineOpcode returns /// true. It returns the MachineInstr opcode value that the node's opcode /// corresponds to. unsigned getMachineOpcode() const { @@ -439,50 +417,44 @@ public: return ~NodeType; } - /// getHasDebugValue - get this bit. + /// Get this bit. bool getHasDebugValue() const { return HasDebugValue; } - /// setHasDebugValue - set this bit. + /// Set this bit. void setHasDebugValue(bool b) { HasDebugValue = b; } - /// use_empty - Return true if there are no uses of this node. - /// + /// Return true if there are no uses of this node. bool use_empty() const { return UseList == nullptr; } - /// hasOneUse - Return true if there is exactly one use of this node. - /// + /// Return true if there is exactly one use of this node. bool hasOneUse() const { return !use_empty() && std::next(use_begin()) == use_end(); } - /// use_size - Return the number of uses of this node. This method takes + /// Return the number of uses of this node. This method takes /// time proportional to the number of uses. - /// size_t use_size() const { return std::distance(use_begin(), use_end()); } - /// getNodeId - Return the unique node id. - /// + /// Return the unique node id. int getNodeId() const { return NodeId; } - /// setNodeId - Set unique node id. + /// Set unique node id. void setNodeId(int Id) { NodeId = Id; } - /// getIROrder - Return the node ordering. - /// + /// Return the node ordering. unsigned getIROrder() const { return IROrder; } - /// setIROrder - Set the node ordering. - /// + /// Set the node ordering. void setIROrder(unsigned Order) { IROrder = Order; } - /// getDebugLoc - Return the source location info. - const DebugLoc getDebugLoc() const { return debugLoc; } + /// Return the source location info. + const DebugLoc &getDebugLoc() const { return debugLoc; } - /// setDebugLoc - Set source location info. Try to avoid this, putting + /// Set source location info. Try to avoid this, putting /// it in the constructor is preferable. - void setDebugLoc(const DebugLoc dl) { debugLoc = dl; } + void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } - /// use_iterator - This class provides iterator support for SDUse + /// This class provides iterator support for SDUse /// operands that use a specific SDNode. class use_iterator : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> { @@ -506,7 +478,7 @@ public: return !operator==(x); } - /// atEnd - return true if this iterator is at the end of uses list. + /// Return true if this iterator is at the end of uses list. bool atEnd() const { return Op == nullptr; } // Iterator traversal: forward iteration only. @@ -530,17 +502,14 @@ public: SDUse &getUse() const { return *Op; } - /// getOperandNo - Retrieve the operand # of this use in its user. - /// + /// Retrieve the operand # of this use in its user. unsigned getOperandNo() const { assert(Op && "Cannot dereference end iterator!"); return (unsigned)(Op - Op->getUser()->OperandList); } }; - /// use_begin/use_end - Provide iteration support to walk over all uses - /// of an SDNode. - + /// Provide iteration support to walk over all uses of an SDNode. use_iterator use_begin() const { return use_iterator(UseList); } @@ -554,37 +523,34 @@ public: return iterator_range<use_iterator>(use_begin(), use_end()); } - /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the - /// indicated value. This method ignores uses of other values defined by this - /// operation. + /// Return true if there are exactly NUSES uses of the indicated value. + /// This method ignores uses of other values defined by this operation. bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; - /// hasAnyUseOfValue - Return true if there are any use of the indicated - /// value. This method ignores uses of other values defined by this operation. + /// Return true if there are any use of the indicated value. + /// This method ignores uses of other values defined by this operation. bool hasAnyUseOfValue(unsigned Value) const; - /// isOnlyUserOf - Return true if this node is the only use of N. - /// + /// Return true if this node is the only use of N. bool isOnlyUserOf(SDNode *N) const; - /// isOperandOf - Return true if this node is an operand of N. - /// + /// Return true if this node is an operand of N. bool isOperandOf(SDNode *N) const; - /// isPredecessorOf - Return true if this node is a predecessor of N. + /// Return true if this node is a predecessor of N. /// NOTE: Implemented on top of hasPredecessor and every bit as /// expensive. Use carefully. bool isPredecessorOf(const SDNode *N) const { return N->hasPredecessor(this); } - /// hasPredecessor - Return true if N is a predecessor of this node. + /// Return true if N is a predecessor of this node. /// N is either an operand of this node, or can be reached by recursively /// traversing up the operands. /// NOTE: This is an expensive method. Use it carefully. bool hasPredecessor(const SDNode *N) const; - /// hasPredecesorHelper - Return true if N is a predecessor of this node. + /// Return true if N is a predecessor of this node. /// N is either an operand of this node, or can be reached by recursively /// traversing up the operands. /// In this helper the Visited and worklist sets are held externally to @@ -597,12 +563,10 @@ public: SmallPtrSetImpl<const SDNode *> &Visited, SmallVectorImpl<const SDNode *> &Worklist) const; - /// getNumOperands - Return the number of values used by this operation. - /// + /// Return the number of values used by this operation. unsigned getNumOperands() const { return NumOperands; } - /// getConstantOperandVal - Helper method returns the integer value of a - /// ConstantSDNode operand. + /// Helper method returns the integer value of a ConstantSDNode operand. uint64_t getConstantOperandVal(unsigned Num) const; const SDValue &getOperand(unsigned Num) const { @@ -620,7 +584,7 @@ public: return X; } - /// getGluedNode - If this node has a glue operand, return the node + /// If this node has a glue operand, return the node /// to which the glue operand points. Otherwise return NULL. SDNode *getGluedNode() const { if (getNumOperands() != 0 && @@ -645,7 +609,7 @@ public: return FoundNode; } - /// getGluedUser - If this node has a glue value with a user, return + /// If this node has a glue value with a user, return /// the user (there is at most one). Otherwise return NULL. SDNode *getGluedUser() const { for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) @@ -654,26 +618,21 @@ public: return nullptr; } - /// getNumValues - Return the number of values defined/returned by this - /// operator. - /// + /// Return the number of values defined/returned by this operator. unsigned getNumValues() const { return NumValues; } - /// getValueType - Return the type of a specified result. - /// + /// Return the type of a specified result. EVT getValueType(unsigned ResNo) const { assert(ResNo < NumValues && "Illegal result number!"); return ValueList[ResNo]; } /// Return the type of a specified result as a simple type. - /// MVT getSimpleValueType(unsigned ResNo) const { return getValueType(ResNo).getSimpleVT(); } - /// getValueSizeInBits - Returns MVT::getSizeInBits(getValueType(ResNo)). - /// + /// Returns MVT::getSizeInBits(getValueType(ResNo)). unsigned getValueSizeInBits(unsigned ResNo) const { return getValueType(ResNo).getSizeInBits(); } @@ -682,8 +641,7 @@ public: value_iterator value_begin() const { return ValueList; } value_iterator value_end() const { return ValueList+NumValues; } - /// getOperationName - Return the opcode of this operation for printing. - /// + /// Return the opcode of this operation for printing. std::string getOperationName(const SelectionDAG *G = nullptr) const; static const char* getIndexedModeName(ISD::MemIndexedMode AM); void print_types(raw_ostream &OS, const SelectionDAG *G) const; @@ -691,7 +649,7 @@ public: void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; - /// printrFull - Print a SelectionDAG node and all children down to + /// Print a SelectionDAG node and all children down to /// the leaves. The given SelectionDAG allows target-specific nodes /// to be printed in human-readable form. Unlike printr, this will /// print the whole DAG, including children that appear multiple @@ -699,7 +657,7 @@ public: /// void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; - /// printrWithDepth - Print a SelectionDAG node and children up to + /// Print a SelectionDAG node and children up to /// depth "depth." The given SelectionDAG allows target-specific /// nodes to be printed in human-readable form. Unlike printr, this /// will print children that appear multiple times wherever they are @@ -709,30 +667,29 @@ public: unsigned depth = 100) const; - /// dump - Dump this node, for debugging. + /// Dump this node, for debugging. void dump() const; - /// dumpr - Dump (recursively) this node and its use-def subgraph. + /// Dump (recursively) this node and its use-def subgraph. void dumpr() const; - /// dump - Dump this node, for debugging. + /// Dump this node, for debugging. /// The given SelectionDAG allows target-specific nodes to be printed /// in human-readable form. void dump(const SelectionDAG *G) const; - /// dumpr - Dump (recursively) this node and its use-def subgraph. + /// Dump (recursively) this node and its use-def subgraph. /// The given SelectionDAG allows target-specific nodes to be printed /// in human-readable form. void dumpr(const SelectionDAG *G) const; - /// dumprFull - printrFull to dbgs(). The given SelectionDAG allows + /// printrFull to dbgs(). The given SelectionDAG allows /// target-specific nodes to be printed in human-readable form. /// Unlike dumpr, this will print the whole DAG, including children /// that appear multiple times. - /// void dumprFull(const SelectionDAG *G = nullptr) const; - /// dumprWithDepth - printrWithDepth to dbgs(). The given + /// printrWithDepth to dbgs(). The given /// SelectionDAG allows target-specific nodes to be printed in /// human-readable form. Unlike dumpr, this will print children /// that appear multiple times wherever they are used. @@ -740,12 +697,10 @@ public: void dumprWithDepth(const SelectionDAG *G = nullptr, unsigned depth = 100) const; - /// Profile - Gather unique data for the node. - /// + /// Gather unique data for the node. void Profile(FoldingSetNodeID &ID) const; - /// addUse - This method should only be used by the SDUse class. - /// + /// This method should only be used by the SDUse class. void addUse(SDUse &U) { U.addToList(&UseList); } protected: @@ -754,14 +709,13 @@ protected: return Ret; } - SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs, + SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, ArrayRef<SDValue> Ops) - : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), - SubclassData(0), NodeId(-1), - OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr), - ValueList(VTs.VTs), UseList(nullptr), - NumOperands(Ops.size()), NumValues(VTs.NumVTs), - debugLoc(dl), IROrder(Order) { + : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), + SubclassData(0), NodeId(-1), + OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr), + ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()), + NumValues(VTs.NumVTs), debugLoc(std::move(dl)), IROrder(Order) { assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumOperands == Ops.size() && "NumOperands wasn't wide enough for its operands!"); @@ -777,17 +731,17 @@ protected: /// This constructor adds no operands itself; operands can be /// set later with InitOperands. - SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs) - : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), - SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), - UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), debugLoc(dl), - IROrder(Order) { + SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) + : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), + SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), + debugLoc(std::move(dl)), IROrder(Order) { assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); } - /// InitOperands - Initialize the operands list of this with 1 operand. + /// Initialize the operands list of this with 1 operand. void InitOperands(SDUse *Ops, const SDValue &Op0) { Ops[0].setUser(this); Ops[0].setInitial(Op0); @@ -796,7 +750,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 2 operands. + /// Initialize the operands list of this with 2 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) { Ops[0].setUser(this); Ops[0].setInitial(Op0); @@ -807,7 +761,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 3 operands. + /// Initialize the operands list of this with 3 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, const SDValue &Op2) { Ops[0].setUser(this); @@ -821,7 +775,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 4 operands. + /// Initialize the operands list of this with 4 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, const SDValue &Op2, const SDValue &Op3) { Ops[0].setUser(this); @@ -837,7 +791,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with N operands. + /// Initialize the operands list of this with N operands. void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) { for (unsigned i = 0; i != N; ++i) { Ops[i].setUser(this); @@ -850,8 +804,7 @@ protected: checkForCycles(this); } - /// DropOperands - Release the operands and set this node to have - /// zero operands. + /// Release the operands and set this node to have zero operands. void DropOperands(); }; @@ -945,7 +898,7 @@ inline bool SDValue::use_empty() const { inline bool SDValue::hasOneUse() const { return Node->hasNUsesOfValue(1, ResNo); } -inline const DebugLoc SDValue::getDebugLoc() const { +inline const DebugLoc &SDValue::getDebugLoc() const { return Node->getDebugLoc(); } inline void SDValue::dump() const { @@ -973,7 +926,63 @@ inline void SDUse::setNode(SDNode *N) { if (N) N->addUse(*this); } -/// UnarySDNode - This class is used for single-operand SDNodes. This is solely +/// These are IR-level optimization flags that may be propagated to SDNodes. +/// TODO: This data structure should be shared by the IR optimizer and the +/// the backend. +struct SDNodeFlags { +private: + bool NoUnsignedWrap : 1; + bool NoSignedWrap : 1; + bool Exact : 1; + bool UnsafeAlgebra : 1; + bool NoNaNs : 1; + bool NoInfs : 1; + bool NoSignedZeros : 1; + bool AllowReciprocal : 1; + +public: + /// Default constructor turns off all optimization flags. + SDNodeFlags() { + NoUnsignedWrap = false; + NoSignedWrap = false; + Exact = false; + UnsafeAlgebra = false; + NoNaNs = false; + NoInfs = false; + NoSignedZeros = false; + AllowReciprocal = false; + } + + // These are mutators for each flag. + void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } + void setNoSignedWrap(bool b) { NoSignedWrap = b; } + void setExact(bool b) { Exact = b; } + void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; } + void setNoNaNs(bool b) { NoNaNs = b; } + void setNoInfs(bool b) { NoInfs = b; } + void setNoSignedZeros(bool b) { NoSignedZeros = b; } + void setAllowReciprocal(bool b) { AllowReciprocal = b; } + + // These are accessors for each flag. + bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } + bool hasNoSignedWrap() const { return NoSignedWrap; } + bool hasExact() const { return Exact; } + bool hasUnsafeAlgebra() const { return UnsafeAlgebra; } + bool hasNoNaNs() const { return NoNaNs; } + bool hasNoInfs() const { return NoInfs; } + bool hasNoSignedZeros() const { return NoSignedZeros; } + bool hasAllowReciprocal() const { return AllowReciprocal; } + + /// Return a raw encoding of the flags. + /// This function should only be used to add data to the NodeID value. + unsigned getRawFlags() const { + return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) | + (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) | + (NoSignedZeros << 6) | (AllowReciprocal << 7); + } +}; + +/// This class is used for single-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class UnarySDNode : public SDNode { SDUse Op; @@ -985,7 +994,7 @@ public: } }; -/// BinarySDNode - This class is used for two-operand SDNodes. This is solely +/// This class is used for two-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class BinarySDNode : public SDNode { SDUse Ops[2]; @@ -997,37 +1006,37 @@ public: } }; -/// BinaryWithFlagsSDNode - This class is an extension of BinarySDNode +/// Returns true if the opcode is a binary operation with flags. +static bool isBinOpWithFlags(unsigned Opcode) { + switch (Opcode) { + case ISD::SDIV: + case ISD::UDIV: + case ISD::SRA: + case ISD::SRL: + case ISD::MUL: + case ISD::ADD: + case ISD::SUB: + case ISD::SHL: + return true; + default: + return false; + } +} + +/// This class is an extension of BinarySDNode /// used from those opcodes that have associated extra flags. class BinaryWithFlagsSDNode : public BinarySDNode { - enum { NUW = (1 << 0), NSW = (1 << 1), EXACT = (1 << 2) }; - public: + SDNodeFlags Flags; BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, SDValue X, SDValue Y) - : BinarySDNode(Opc, Order, dl, VTs, X, Y) {} - /// getRawSubclassData - Return the SubclassData value, which contains an - /// encoding of the flags. - /// This function should be used to add subclass data to the NodeID value. - unsigned getRawSubclassData() const { return SubclassData; } - void setHasNoUnsignedWrap(bool b) { - SubclassData = (SubclassData & ~NUW) | (b ? NUW : 0); - } - void setHasNoSignedWrap(bool b) { - SubclassData = (SubclassData & ~NSW) | (b ? NSW : 0); - } - void setIsExact(bool b) { - SubclassData = (SubclassData & ~EXACT) | (b ? EXACT : 0); - } - bool hasNoUnsignedWrap() const { return SubclassData & NUW; } - bool hasNoSignedWrap() const { return SubclassData & NSW; } - bool isExact() const { return SubclassData & EXACT; } + : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags() {} static bool classof(const SDNode *N) { return isBinOpWithFlags(N->getOpcode()); } }; -/// TernarySDNode - This class is used for three-operand SDNodes. This is solely +/// This class is used for three-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class TernarySDNode : public SDNode { SDUse Ops[3]; @@ -1040,7 +1049,7 @@ public: }; -/// HandleSDNode - This class is used to form a handle around another node that +/// This class is used to form a handle around another node that /// is persistent and is updated across invocations of replaceAllUsesWith on its /// operand. This node should be directly created by end-users and not added to /// the AllNodes list. @@ -1072,14 +1081,14 @@ public: } }; -/// Abstact virtual class for operations for memory operations +/// This is an abstract virtual class for memory operations. class MemSDNode : public SDNode { private: - // MemoryVT - VT of in-memory value. + // VT of in-memory value. EVT MemoryVT; protected: - /// MMO - Memory reference information. + /// Memory reference information. MachineMemOperand *MMO; public: @@ -1100,7 +1109,7 @@ public: return MMO->getAlignment(); } - /// getRawSubclassData - Return the SubclassData value, which contains an + /// Return the SubclassData value, which contains an /// encoding of the volatile flag, as well as bits used by subclasses. This /// function should only be used to compute a FoldingSetNodeID value. unsigned getRawSubclassData() const { @@ -1129,10 +1138,10 @@ public: /// Returns the Ranges that describes the dereference. const MDNode *getRanges() const { return MMO->getRanges(); } - /// getMemoryVT - Return the type of the in-memory value. + /// Return the type of the in-memory value. EVT getMemoryVT() const { return MemoryVT; } - /// getMemOperand - Return a MachineMemOperand object describing the memory + /// Return a MachineMemOperand object describing the memory /// reference performed by operation. MachineMemOperand *getMemOperand() const { return MMO; } @@ -1140,12 +1149,12 @@ public: return MMO->getPointerInfo(); } - /// getAddressSpace - Return the address space for the associated pointer + /// Return the address space for the associated pointer unsigned getAddressSpace() const { return getPointerInfo().getAddrSpace(); } - /// refineAlignment - Update this MemSDNode's MachineMemOperand information + /// Update this MemSDNode's MachineMemOperand information /// to reflect the alignment of NewMMO, if it has a greater alignment. /// This must only be used when the new alignment applies to all users of /// this MachineMemOperand. @@ -1182,13 +1191,14 @@ public: N->getOpcode() == ISD::ATOMIC_STORE || N->getOpcode() == ISD::MLOAD || N->getOpcode() == ISD::MSTORE || + N->getOpcode() == ISD::MGATHER || + N->getOpcode() == ISD::MSCATTER || N->isMemIntrinsic() || N->isTargetMemoryOpcode(); } }; -/// AtomicSDNode - A SDNode reprenting atomic operations. -/// +/// This is an SDNode representing atomic operations. class AtomicSDNode : public MemSDNode { SDUse Ops[4]; @@ -1301,7 +1311,7 @@ public: } }; -/// MemIntrinsicSDNode - This SDNode is used for target intrinsics that touch +/// This SDNode is used for target intrinsics that touch /// memory and need an associated MachineMemOperand. Its opcode may be /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. @@ -1324,7 +1334,7 @@ public: } }; -/// ShuffleVectorSDNode - This SDNode is used to implement the code generator +/// This SDNode is used to implement the code generator /// support for the llvm IR shufflevector instruction. It combines elements /// from two input vectors into a new input vector, with the selection and /// ordering of elements determined by an array of integers, referred to as @@ -1368,6 +1378,21 @@ public: } static bool isSplatMask(const int *Mask, EVT VT); + /// Change values in a shuffle permute mask assuming + /// the two vector operands have swapped position. + static void commuteMask(SmallVectorImpl<int> &Mask) { + unsigned NumElems = Mask.size(); + for (unsigned i = 0; i != NumElems; ++i) { + int idx = Mask[i]; + if (idx < 0) + continue; + else if (idx < (int)NumElems) + Mask[i] = idx + NumElems; + else + Mask[i] = idx - NumElems; + } + } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::VECTOR_SHUFFLE; } @@ -1376,9 +1401,10 @@ public: class ConstantSDNode : public SDNode { const ConstantInt *Value; friend class SelectionDAG; - ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) + ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, + DebugLoc DL, EVT VT) : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, - 0, DebugLoc(), getSDVTList(VT)), Value(val) { + 0, DL, getSDVTList(VT)), Value(val) { SubclassData |= (uint16_t)isOpaque; } public: @@ -1403,28 +1429,28 @@ public: class ConstantFPSDNode : public SDNode { const ConstantFP *Value; friend class SelectionDAG; - ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) + ConstantFPSDNode(bool isTarget, const ConstantFP *val, DebugLoc DL, EVT VT) : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, - 0, DebugLoc(), getSDVTList(VT)), Value(val) { + 0, DL, getSDVTList(VT)), Value(val) { } public: const APFloat& getValueAPF() const { return Value->getValueAPF(); } const ConstantFP *getConstantFPValue() const { return Value; } - /// isZero - Return true if the value is positive or negative zero. + /// Return true if the value is positive or negative zero. bool isZero() const { return Value->isZero(); } - /// isNaN - Return true if the value is a NaN. + /// Return true if the value is a NaN. bool isNaN() const { return Value->isNaN(); } - /// isInfinity - Return true if the value is an infinity + /// Return true if the value is an infinity bool isInfinity() const { return Value->isInfinity(); } - /// isNegative - Return true if the value is negative. + /// Return true if the value is negative. bool isNegative() const { return Value->isNegative(); } - /// isExactlyValue - We don't rely on operator== working on double values, as + /// We don't rely on operator== working on double values, as /// it returns true for things that are clearly not equal, like -0.0 and 0.0. /// As such, this method can be used to do an exact bit-for-bit comparison of /// two floating point values. @@ -1608,13 +1634,12 @@ public: } }; -/// BuildVectorSDNode - A "pseudo-class" with methods for operating on -/// BUILD_VECTORs. +/// A "pseudo-class" with methods for operating on BUILD_VECTORs. class BuildVectorSDNode : public SDNode { // These are constructed as SDNodes and then cast to BuildVectorSDNodes. - explicit BuildVectorSDNode() LLVM_DELETED_FUNCTION; + explicit BuildVectorSDNode() = delete; public: - /// isConstantSplat - Check if this is a constant splat, and if so, find the + /// Check if this is a constant splat, and if so, find the /// smallest element size that splats the vector. If MinSplatBits is /// nonzero, the element size must be at least that large. Note that the /// splat element may be the entire vector (i.e., a one element vector). @@ -1657,7 +1682,7 @@ public: } }; -/// SrcValueSDNode - An SDNode that holds an arbitrary LLVM IR Value. This is +/// An SDNode that holds an arbitrary LLVM IR Value. This is /// used when the SelectionDAG needs to make a simple reference to something /// in the LLVM IR representation. /// @@ -1669,7 +1694,7 @@ class SrcValueSDNode : public SDNode { : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} public: - /// getValue - return the contained Value. + /// Return the contained Value. const Value *getValue() const { return V; } static bool classof(const SDNode *N) { @@ -1796,7 +1821,7 @@ public: } }; -/// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the +/// NOTE: avoid using this node as this may disappear in the /// future and most targets don't support it. class CvtRndSatSDNode : public SDNode { ISD::CvtCode CvtCode; @@ -1815,7 +1840,7 @@ public: } }; -/// VTSDNode - This class is used to represent EVT's, which are used +/// This class is used to represent EVT's, which are used /// to parameterize some operations. class VTSDNode : public SDNode { EVT ValueType; @@ -1833,8 +1858,7 @@ public: } }; -/// LSBaseSDNode - Base class for LoadSDNode and StoreSDNode -/// +/// Base class for LoadSDNode and StoreSDNode class LSBaseSDNode : public MemSDNode { //! Operand array for load and store /*! @@ -1860,16 +1884,16 @@ public: return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); } - /// getAddressingMode - Return the addressing mode for this load or store: + /// Return the addressing mode for this load or store: /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. ISD::MemIndexedMode getAddressingMode() const { return ISD::MemIndexedMode((SubclassData >> 2) & 7); } - /// isIndexed - Return true if this is a pre/post inc/dec load/store. + /// Return true if this is a pre/post inc/dec load/store. bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } - /// isUnindexed - Return true if this is NOT a pre/post inc/dec load/store. + /// Return true if this is NOT a pre/post inc/dec load/store. bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } static bool classof(const SDNode *N) { @@ -1878,8 +1902,7 @@ public: } }; -/// LoadSDNode - This class is used to represent ISD::LOAD nodes. -/// +/// This class is used to represent ISD::LOAD nodes. class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs, @@ -1893,7 +1916,7 @@ class LoadSDNode : public LSBaseSDNode { } public: - /// getExtensionType - Return whether this is a plain node, + /// Return whether this is a plain node, /// or one of the varieties of value-extending loads. ISD::LoadExtType getExtensionType() const { return ISD::LoadExtType(SubclassData & 3); @@ -1907,8 +1930,7 @@ public: } }; -/// StoreSDNode - This class is used to represent ISD::STORE nodes. -/// +/// This class is used to represent ISD::STORE nodes. class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl, @@ -1923,7 +1945,7 @@ class StoreSDNode : public LSBaseSDNode { } public: - /// isTruncatingStore - Return true if the op does a truncation before store. + /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. bool isTruncatingStore() const { return SubclassData & 1; } @@ -1937,9 +1959,7 @@ public: } }; -/// MaskedLoadStoreSDNode - This is a base class is used to represent MLOAD and -/// MSTORE nodes -/// +/// This base class is used to represent MLOAD and MSTORE nodes class MaskedLoadStoreSDNode : public MemSDNode { // Operands SDUse Ops[4]; @@ -1965,8 +1985,7 @@ public: } }; -/// MaskedLoadSDNode - This class is used to represent an MLOAD node -/// +/// This class is used to represent an MLOAD node class MaskedLoadSDNode : public MaskedLoadStoreSDNode { public: friend class SelectionDAG; @@ -1987,8 +2006,7 @@ public: } }; -/// MaskedStoreSDNode - This class is used to represent an MSTORE node -/// +/// This class is used to represent an MSTORE node class MaskedStoreSDNode : public MaskedLoadStoreSDNode { public: @@ -2000,7 +2018,7 @@ public: VTs, MemVT, MMO) { SubclassData |= (unsigned short)isTrunc; } - /// isTruncatingStore - Return true if the op does a truncation before store. + /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. bool isTruncatingStore() const { return SubclassData & 1; } @@ -2012,10 +2030,85 @@ public: } }; -/// MachineSDNode - An SDNode that represents everything that will be needed +/// This is a base class used to represent +/// MGATHER and MSCATTER nodes +/// +class MaskedGatherScatterSDNode : public MemSDNode { + // Operands + SDUse Ops[5]; +public: + friend class SelectionDAG; + MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT, + MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + assert(Operands.size() == 5 && "Incompatible number of operands"); + InitOperands(Ops, Operands.data(), Operands.size()); + } + + // In the both nodes address is Op1, mask is Op2: + // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value + // MaskedScatterSDNode (Chain, value, mask, base, index) + // Mask is a vector of i1 elements + const SDValue &getBasePtr() const { return getOperand(3); } + const SDValue &getIndex() const { return getOperand(4); } + const SDValue &getMask() const { return getOperand(2); } + const SDValue &getValue() const { return getOperand(1); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MGATHER || + N->getOpcode() == ISD::MSCATTER; + } +}; + +/// This class is used to represent an MGATHER node +/// +class MaskedGatherSDNode : public MaskedGatherScatterSDNode { +public: + friend class SelectionDAG; + MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getValue().getValueType() == getValueType(0) && + "Incompatible type of the PathThru value in MaskedGatherSDNode"); + assert(getMask().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getMask().getValueType().getScalarType() == MVT::i1 && + "Vector width mismatch between mask and data"); + } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MGATHER; + } +}; + +/// This class is used to represent an MSCATTER node +/// +class MaskedScatterSDNode : public MaskedGatherScatterSDNode { + +public: + friend class SelectionDAG; + MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getMask().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getMask().getValueType().getScalarType() == MVT::i1 && + "Vector width mismatch between mask and data"); + } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MSCATTER; + } +}; + +/// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. -/// class MachineSDNode : public SDNode { public: typedef MachineMemOperand **mmo_iterator; @@ -2025,11 +2118,11 @@ private: MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {} - /// LocalOperands - Operands for this instruction, if they fit here. If + /// Operands for this instruction, if they fit here. If /// they don't, this field is unused. SDUse LocalOperands[4]; - /// MemRefs - Memory reference descriptions for this instruction. + /// Memory reference descriptions for this instruction. mmo_iterator MemRefs; mmo_iterator MemRefsEnd; @@ -2038,7 +2131,7 @@ public: mmo_iterator memoperands_end() const { return MemRefsEnd; } bool memoperands_empty() const { return MemRefsEnd == MemRefs; } - /// setMemRefs - Assign this MachineSDNodes's memory reference descriptor + /// Assign this MachineSDNodes's memory reference descriptor /// list. This does not transfer ownership. void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI) @@ -2064,12 +2157,6 @@ public: } bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } - const SDNodeIterator &operator=(const SDNodeIterator &I) { - assert(I.Node == Node && "Cannot assign iterators to two different nodes!"); - Operand = I.Operand; - return *this; - } - pointer operator*() const { return Node->getOperand(Operand).getNode(); } @@ -2109,60 +2196,51 @@ template <> struct GraphTraits<SDNode*> { } }; -/// LargestSDNode - The largest SDNode class. -/// -typedef AtomicSDNode LargestSDNode; +/// The largest SDNode class. +typedef MaskedGatherScatterSDNode LargestSDNode; -/// MostAlignedSDNode - The SDNode class with the greatest alignment -/// requirement. -/// +/// The SDNode class with the greatest alignment requirement. typedef GlobalAddressSDNode MostAlignedSDNode; namespace ISD { - /// isNormalLoad - Returns true if the specified node is a non-extending - /// and unindexed load. + /// Returns true if the specified node is a non-extending and unindexed load. inline bool isNormalLoad(const SDNode *N) { const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && Ld->getAddressingMode() == ISD::UNINDEXED; } - /// isNON_EXTLoad - Returns true if the specified node is a non-extending - /// load. + /// Returns true if the specified node is a non-extending load. inline bool isNON_EXTLoad(const SDNode *N) { return isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; } - /// isEXTLoad - Returns true if the specified node is a EXTLOAD. - /// + /// Returns true if the specified node is a EXTLOAD. inline bool isEXTLoad(const SDNode *N) { return isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; } - /// isSEXTLoad - Returns true if the specified node is a SEXTLOAD. - /// + /// Returns true if the specified node is a SEXTLOAD. inline bool isSEXTLoad(const SDNode *N) { return isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; } - /// isZEXTLoad - Returns true if the specified node is a ZEXTLOAD. - /// + /// Returns true if the specified node is a ZEXTLOAD. inline bool isZEXTLoad(const SDNode *N) { return isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; } - /// isUNINDEXEDLoad - Returns true if the specified node is an unindexed load. - /// + /// Returns true if the specified node is an unindexed load. inline bool isUNINDEXEDLoad(const SDNode *N) { return isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; } - /// isNormalStore - Returns true if the specified node is a non-truncating + /// Returns true if the specified node is a non-truncating /// and unindexed store. inline bool isNormalStore(const SDNode *N) { const StoreSDNode *St = dyn_cast<StoreSDNode>(N); @@ -2170,20 +2248,17 @@ namespace ISD { St->getAddressingMode() == ISD::UNINDEXED; } - /// isNON_TRUNCStore - Returns true if the specified node is a non-truncating - /// store. + /// Returns true if the specified node is a non-truncating store. inline bool isNON_TRUNCStore(const SDNode *N) { return isa<StoreSDNode>(N) && !cast<StoreSDNode>(N)->isTruncatingStore(); } - /// isTRUNCStore - Returns true if the specified node is a truncating - /// store. + /// Returns true if the specified node is a truncating store. inline bool isTRUNCStore(const SDNode *N) { return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->isTruncatingStore(); } - /// isUNINDEXEDStore - Returns true if the specified node is an - /// unindexed store. + /// Returns true if the specified node is an unindexed store. inline bool isUNINDEXEDStore(const SDNode *N) { return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 00bb22b91d32c..9d6d6f5b1be0e 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -162,7 +162,7 @@ namespace llvm { } /// Return true for a valid index. - LLVM_EXPLICIT operator bool() const { return isValid(); } + explicit operator bool() const { return isValid(); } /// Print this index to the given raw_ostream. void print(raw_ostream &os) const; diff --git a/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/include/llvm/CodeGen/StackMapLivenessAnalysis.h deleted file mode 100644 index f67a6e95191d8..0000000000000 --- a/include/llvm/CodeGen/StackMapLivenessAnalysis.h +++ /dev/null @@ -1,64 +0,0 @@ -//===--- StackMapLivenessAnalysis - StackMap Liveness Analysis --*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass calculates the liveness for each basic block in a function and -// attaches the register live-out information to a patchpoint intrinsic (if -// present). -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H -#define LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H - -#include "llvm/CodeGen/LivePhysRegs.h" -#include "llvm/CodeGen/MachineFunctionPass.h" - - -namespace llvm { - -/// \brief This pass calculates the liveness information for each basic block in -/// a function and attaches the register live-out information to a patchpoint -/// intrinsic if present. -/// -/// This pass can be disabled via the -enable-patchpoint-liveness=false flag. -/// The pass skips functions that don't have any patchpoint intrinsics. The -/// information provided by this pass is optional and not required by the -/// aformentioned intrinsic to function. -class StackMapLiveness : public MachineFunctionPass { - MachineFunction *MF; - const TargetRegisterInfo *TRI; - LivePhysRegs LiveRegs; -public: - static char ID; - - /// \brief Default construct and initialize the pass. - StackMapLiveness(); - - /// \brief Tell the pass manager which passes we depend on and what - /// information we preserve. - void getAnalysisUsage(AnalysisUsage &AU) const override; - - /// \brief Calculate the liveness information for the given machine function. - bool runOnMachineFunction(MachineFunction &MF) override; - -private: - /// \brief Performs the actual liveness calculation for the function. - bool calculateLiveness(); - - /// \brief Add the current register live set to the instruction. - void addLiveOutSetToMI(MachineInstr &MI); - - /// \brief Create a register mask and initialize it with the registers from - /// the register live set. - uint32_t *createRegisterMask() const; -}; - -} // llvm namespace - -#endif diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index 4e48afe140042..46a773f74aaca 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -14,6 +14,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/Support/Debug.h" #include <map> #include <vector> @@ -84,15 +85,23 @@ public: /// MI-level Statepoint operands /// /// Statepoint operands take the form: -/// <num call arguments>, <call target>, [call arguments], -/// <StackMaps::ConstantOp>, <flags>, +/// <id>, <num patch bytes >, <num call arguments>, <call target>, +/// [call arguments], <StackMaps::ConstantOp>, <calling convention>, +/// <StackMaps::ConstantOp>, <statepoint flags>, /// <StackMaps::ConstantOp>, <num other args>, [other args], /// [gc values] class StatepointOpers { private: + // These values are aboolute offsets into the operands of the statepoint + // instruction. + enum { IDPos, NBytesPos, NCallArgsPos, CallTargetPos, MetaEnd }; + + // These values are relative offests from the start of the statepoint meta + // arguments (i.e. the end of the call arguments). enum { - NCallArgsPos = 0, - CallTargetPos = 1 + CCOffset = 1, + FlagsOffset = 3, + NumVMSArgsOffset = 5 }; public: @@ -100,22 +109,17 @@ public: MI(MI) { } /// Get starting index of non call related arguments - /// (statepoint flags, vm state and gc state). + /// (calling convention, statepoint flags, vm state and gc state). unsigned getVarIdx() const { - return MI->getOperand(NCallArgsPos).getImm() + 2; + return MI->getOperand(NCallArgsPos).getImm() + MetaEnd; } - /// Returns the index of the operand containing the number of non-gc non-call - /// arguments. - unsigned getNumVMSArgsIdx() const { - return getVarIdx() + 3; - } + /// Return the ID for the given statepoint. + uint64_t getID() const { return MI->getOperand(IDPos).getImm(); } - /// Returns the number of non-gc non-call arguments attached to the - /// statepoint. Note that this is the number of arguments, not the number of - /// operands required to represent those arguments. - unsigned getNumVMSArgs() const { - return MI->getOperand(getNumVMSArgsIdx()).getImm(); + /// Return the number of patchable bytes the given statepoint should emit. + uint32_t getNumPatchBytes() const { + return MI->getOperand(NBytesPos).getImm(); } /// Returns the target of the underlying call. @@ -245,7 +249,10 @@ private: void emitConstantPoolEntries(MCStreamer &OS); /// \brief Emit the callsite info for each stackmap/patchpoint intrinsic call. - void emitCallsiteEntries(MCStreamer &OS, const TargetRegisterInfo *TRI); + void emitCallsiteEntries(MCStreamer &OS); + + void print(raw_ostream &OS); + void debug() { print(dbgs()); } }; } diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 9209e1c67c1bb..9a1b63f91bd46 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -34,25 +34,34 @@ namespace llvm { class TargetLoweringObjectFileELF : public TargetLoweringObjectFile { bool UseInitArray; + mutable unsigned NextUniqueID = 0; public: - virtual ~TargetLoweringObjectFileELF() {} + TargetLoweringObjectFileELF() : UseInitArray(false) {} + + ~TargetLoweringObjectFileELF() override {} void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM, const MCSymbol *Sym) const override; /// Given a constant with the SectionKind, return a section that it should be /// placed in. - const MCSection *getSectionForConstant(SectionKind Kind, - const Constant *C) const override; + MCSection *getSectionForConstant(SectionKind Kind, + const Constant *C) const override; + + MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; + + MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection *getExplicitSectionGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection *SelectSectionForGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference, + const Function &F) const override; /// Return an MCExpr to use for a reference to the specified type info global /// variable from exception handling information. @@ -68,17 +77,18 @@ public: MachineModuleInfo *MMI) const override; void InitializeELF(bool UseInitArray_); - const MCSection *getStaticCtorSection(unsigned Priority, - const MCSymbol *KeySym) const override; - const MCSection *getStaticDtorSection(unsigned Priority, - const MCSymbol *KeySym) const override; + MCSection *getStaticCtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + MCSection *getStaticDtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; }; class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { public: - virtual ~TargetLoweringObjectFileMachO() {} + ~TargetLoweringObjectFileMachO() override {} + TargetLoweringObjectFileMachO(); /// Extract the dependent library name from a linker option string. Returns /// StringRef() if the option does not specify a library. @@ -89,18 +99,16 @@ public: ArrayRef<Module::ModuleFlagEntry> ModuleFlags, Mangler &Mang, const TargetMachine &TM) const override; - const MCSection * - SelectSectionForGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection * - getExplicitSectionGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection *getSectionForConstant(SectionKind Kind, - const Constant *C) const override; + MCSection *getSectionForConstant(SectionKind Kind, + const Constant *C) const override; /// The mach-o version of this method defaults to returning a stub reference. const MCExpr * @@ -113,23 +121,34 @@ public: MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang, const TargetMachine &TM, MachineModuleInfo *MMI) const override; + + /// Get MachO PC relative GOT entry relocation + const MCExpr *getIndirectSymViaGOTPCRel(const MCSymbol *Sym, + const MCValue &MV, int64_t Offset, + MachineModuleInfo *MMI, + MCStreamer &Streamer) const override; }; class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile { public: - virtual ~TargetLoweringObjectFileCOFF() {} + ~TargetLoweringObjectFileCOFF() override {} + + MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; + + MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, + Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection * - getExplicitSectionGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV, + bool CannotUsePrivateLabel, Mangler &Mang, + const TargetMachine &TM) const override; - const MCSection * - SelectSectionForGlobal(const GlobalValue *GV, - SectionKind Kind, Mangler &Mang, - const TargetMachine &TM) const override; + MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang, + const TargetMachine &TM) const override; /// Extract the dependent library name from a linker option string. Returns /// StringRef() if the option does not specify a library. @@ -141,10 +160,10 @@ public: ArrayRef<Module::ModuleFlagEntry> ModuleFlags, Mangler &Mang, const TargetMachine &TM) const override; - const MCSection *getStaticCtorSection(unsigned Priority, - const MCSymbol *KeySym) const override; - const MCSection *getStaticDtorSection(unsigned Priority, - const MCSymbol *KeySym) const override; + MCSection *getStaticCtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + MCSection *getStaticDtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; }; } // end namespace llvm diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index b6136665b968c..751fac411ce6b 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -40,6 +40,9 @@ class TargetSchedModel { SmallVector<unsigned, 16> ResourceFactors; unsigned MicroOpFactor; // Multiply to normalize microops to resource units. unsigned ResourceLCM; // Resource units per cycle. Latency normalization factor. + + unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const; + public: TargetSchedModel(): SchedModel(MCSchedModel::GetDefaultSchedModel()), STI(nullptr), TII(nullptr) {} diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index 4e93940e223ef..e1a9fd38290ba 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -34,10 +34,9 @@ namespace llvm { Type *LLVMTy; public: - EVT() : V((MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE)), - LLVMTy(nullptr) {} - EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(nullptr) { } - EVT(MVT S) : V(S), LLVMTy(nullptr) {} + LLVM_CONSTEXPR EVT() : V(MVT::INVALID_SIMPLE_VALUE_TYPE), LLVMTy(nullptr) {} + LLVM_CONSTEXPR EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(nullptr) {} + LLVM_CONSTEXPR EVT(MVT S) : V(S), LLVMTy(nullptr) {} bool operator==(EVT VT) const { return !(*this != VT); diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td index b5fa0e8c6a231..2b30f14f902c6 100644 --- a/include/llvm/CodeGen/ValueTypes.td +++ b/include/llvm/CodeGen/ValueTypes.td @@ -62,25 +62,26 @@ def v2i64 : ValueType<128, 38>; // 2 x i64 vector value def v4i64 : ValueType<256, 39>; // 4 x i64 vector value def v8i64 : ValueType<512, 40>; // 8 x i64 vector value def v16i64 : ValueType<1024,41>; // 16 x i64 vector value +def v1i128 : ValueType<128, 42>; // 1 x i128 vector value -def v2f16 : ValueType<32 , 42>; // 2 x f16 vector value -def v4f16 : ValueType<64 , 43>; // 4 x f16 vector value -def v8f16 : ValueType<128, 44>; // 8 x f16 vector value -def v1f32 : ValueType<32 , 45>; // 1 x f32 vector value -def v2f32 : ValueType<64 , 46>; // 2 x f32 vector value -def v4f32 : ValueType<128, 47>; // 4 x f32 vector value -def v8f32 : ValueType<256, 48>; // 8 x f32 vector value -def v16f32 : ValueType<512, 49>; // 16 x f32 vector value -def v1f64 : ValueType<64, 50>; // 1 x f64 vector value -def v2f64 : ValueType<128, 51>; // 2 x f64 vector value -def v4f64 : ValueType<256, 52>; // 4 x f64 vector value -def v8f64 : ValueType<512, 53>; // 8 x f64 vector value +def v2f16 : ValueType<32 , 43>; // 2 x f16 vector value +def v4f16 : ValueType<64 , 44>; // 4 x f16 vector value +def v8f16 : ValueType<128, 45>; // 8 x f16 vector value +def v1f32 : ValueType<32 , 46>; // 1 x f32 vector value +def v2f32 : ValueType<64 , 47>; // 2 x f32 vector value +def v4f32 : ValueType<128, 48>; // 4 x f32 vector value +def v8f32 : ValueType<256, 49>; // 8 x f32 vector value +def v16f32 : ValueType<512, 50>; // 16 x f32 vector value +def v1f64 : ValueType<64, 51>; // 1 x f64 vector value +def v2f64 : ValueType<128, 52>; // 2 x f64 vector value +def v4f64 : ValueType<256, 53>; // 4 x f64 vector value +def v8f64 : ValueType<512, 54>; // 8 x f64 vector value -def x86mmx : ValueType<64 , 54>; // X86 MMX value -def FlagVT : ValueType<0 , 55>; // Pre-RA sched glue -def isVoid : ValueType<0 , 56>; // Produces no value -def untyped: ValueType<8 , 57>; // Produces an untyped value +def x86mmx : ValueType<64 , 55>; // X86 MMX value +def FlagVT : ValueType<0 , 56>; // Pre-RA sched glue +def isVoid : ValueType<0 , 57>; // Produces no value +def untyped: ValueType<8 , 58>; // Produces an untyped value def MetadataVT: ValueType<0, 250>; // Metadata // Pseudo valuetype mapped to the current pointer size to any address space. @@ -98,3 +99,6 @@ def iAny : ValueType<0 , 254>; // Pseudo valuetype mapped to the current pointer size. def iPTR : ValueType<0 , 255>; + +// Pseudo valuetype to represent "any type of any size". +def Any : ValueType<0 , 256>; diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h index eceb8755763e2..d7e92094877d1 100644 --- a/include/llvm/CodeGen/VirtRegMap.h +++ b/include/llvm/CodeGen/VirtRegMap.h @@ -63,8 +63,8 @@ namespace llvm { /// createSpillSlot - Allocate a spill slot for RC from MFI. unsigned createSpillSlot(const TargetRegisterClass *RC); - VirtRegMap(const VirtRegMap&) LLVM_DELETED_FUNCTION; - void operator=(const VirtRegMap&) LLVM_DELETED_FUNCTION; + VirtRegMap(const VirtRegMap&) = delete; + void operator=(const VirtRegMap&) = delete; public: static char ID; diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h new file mode 100644 index 0000000000000..e2644edd4d12a --- /dev/null +++ b/include/llvm/CodeGen/WinEHFuncInfo.h @@ -0,0 +1,157 @@ +//===-- llvm/CodeGen/WinEHFuncInfo.h ----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Data structures and associated state for Windows exception handling schemes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_WINEHFUNCINFO_H +#define LLVM_CODEGEN_WINEHFUNCINFO_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/ADT/DenseMap.h" + +namespace llvm { +class BasicBlock; +class Constant; +class Function; +class GlobalVariable; +class IntrinsicInst; +class LandingPadInst; +class MCSymbol; +class Value; + +enum ActionType { Catch, Cleanup }; + +class ActionHandler { +public: + ActionHandler(BasicBlock *BB, ActionType Type) + : StartBB(BB), Type(Type), EHState(-1), HandlerBlockOrFunc(nullptr) {} + + ActionType getType() const { return Type; } + BasicBlock *getStartBlock() const { return StartBB; } + + bool hasBeenProcessed() { return HandlerBlockOrFunc != nullptr; } + + void setHandlerBlockOrFunc(Constant *F) { HandlerBlockOrFunc = F; } + Constant *getHandlerBlockOrFunc() { return HandlerBlockOrFunc; } + + void setEHState(int State) { EHState = State; } + int getEHState() const { return EHState; } + +private: + BasicBlock *StartBB; + ActionType Type; + int EHState; + + // Can be either a BlockAddress or a Function depending on the EH personality. + Constant *HandlerBlockOrFunc; +}; + +class CatchHandler : public ActionHandler { +public: + CatchHandler(BasicBlock *BB, Constant *Selector, BasicBlock *NextBB) + : ActionHandler(BB, ActionType::Catch), Selector(Selector), + NextBB(NextBB), ExceptionObjectVar(nullptr), + ExceptionObjectIndex(-1) {} + + // Method for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const ActionHandler *H) { + return H->getType() == ActionType::Catch; + } + + Constant *getSelector() const { return Selector; } + BasicBlock *getNextBB() const { return NextBB; } + + const Value *getExceptionVar() { return ExceptionObjectVar; } + TinyPtrVector<BasicBlock *> &getReturnTargets() { return ReturnTargets; } + + void setExceptionVar(const Value *Val) { ExceptionObjectVar = Val; } + void setExceptionVarIndex(int Index) { ExceptionObjectIndex = Index; } + int getExceptionVarIndex() const { return ExceptionObjectIndex; } + void setReturnTargets(TinyPtrVector<BasicBlock *> &Targets) { + ReturnTargets = Targets; + } + +private: + Constant *Selector; + BasicBlock *NextBB; + // While catch handlers are being outlined the ExceptionObjectVar field will + // be populated with the instruction in the parent frame that corresponds + // to the exception object (or nullptr if the catch does not use an + // exception object) and the ExceptionObjectIndex field will be -1. + // When the parseEHActions function is called to populate a vector of + // instances of this class, the ExceptionObjectVar field will be nullptr + // and the ExceptionObjectIndex will be the index of the exception object in + // the parent function's frameescape block. + const Value *ExceptionObjectVar; + int ExceptionObjectIndex; + TinyPtrVector<BasicBlock *> ReturnTargets; +}; + +class CleanupHandler : public ActionHandler { +public: + CleanupHandler(BasicBlock *BB) : ActionHandler(BB, ActionType::Cleanup) {} + + // Method for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const ActionHandler *H) { + return H->getType() == ActionType::Cleanup; + } +}; + +void parseEHActions(const IntrinsicInst *II, + SmallVectorImpl<std::unique_ptr<ActionHandler>> &Actions); + +// The following structs respresent the .xdata for functions using C++ +// exceptions on Windows. + +struct WinEHUnwindMapEntry { + int ToState; + Function *Cleanup; +}; + +struct WinEHHandlerType { + int Adjectives; + GlobalVariable *TypeDescriptor; + int CatchObjRecoverIdx; + Function *Handler; +}; + +struct WinEHTryBlockMapEntry { + int TryLow; + int TryHigh; + SmallVector<WinEHHandlerType, 1> HandlerArray; +}; + +struct WinEHFuncInfo { + DenseMap<const Function *, const LandingPadInst *> RootLPad; + DenseMap<const Function *, const InvokeInst *> LastInvoke; + DenseMap<const Function *, int> HandlerEnclosedState; + DenseMap<const Function *, bool> LastInvokeVisited; + DenseMap<const LandingPadInst *, int> LandingPadStateMap; + DenseMap<const Function *, int> CatchHandlerParentFrameObjIdx; + DenseMap<const Function *, int> CatchHandlerParentFrameObjOffset; + DenseMap<const Function *, int> CatchHandlerMaxState; + DenseMap<const Function *, int> HandlerBaseState; + SmallVector<WinEHUnwindMapEntry, 4> UnwindMap; + SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap; + SmallVector<std::pair<MCSymbol *, int>, 4> IPToStateList; + int UnwindHelpFrameIdx; + int UnwindHelpFrameOffset; + + unsigned NumIPToStateFuncsVisited; + + WinEHFuncInfo() + : UnwindHelpFrameIdx(INT_MAX), UnwindHelpFrameOffset(-1), + NumIPToStateFuncsVisited(0) {} +}; + +} +#endif // LLVM_CODEGEN_WINEHFUNCINFO_H |