diff options
Diffstat (limited to 'include')
469 files changed, 18026 insertions, 6570 deletions
diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h index cac2f297056d..b84970956666 100644 --- a/include/llvm-c/Core.h +++ b/include/llvm-c/Core.h @@ -370,9 +370,13 @@ typedef enum { LLVMAtomicRMWBinOpUMax, /**< Sets the value if it's greater than the original using an unsigned comparison and return the old one */ - LLVMAtomicRMWBinOpUMin /**< Sets the value if it's greater than the - original using an unsigned comparison and return - the old one */ + LLVMAtomicRMWBinOpUMin, /**< Sets the value if it's greater than the + original using an unsigned comparison and return + the old one */ + LLVMAtomicRMWBinOpFAdd, /**< Add a floating point value and return the + old one */ + LLVMAtomicRMWBinOpFSub /**< Subtract a floating point value and return the + old one */ } LLVMAtomicRMWBinOp; typedef enum { @@ -1539,6 +1543,7 @@ LLVMTypeRef LLVMX86MMXType(void); macro(GlobalVariable) \ macro(UndefValue) \ macro(Instruction) \ + macro(UnaryOperator) \ macro(BinaryOperator) \ macro(CallInst) \ macro(IntrinsicInst) \ @@ -1571,6 +1576,8 @@ LLVMTypeRef LLVMX86MMXType(void); macro(ResumeInst) \ macro(CleanupReturnInst) \ macro(CatchReturnInst) \ + macro(CatchSwitchInst) \ + macro(CallBrInst) \ macro(FuncletPadInst) \ macro(CatchPadInst) \ macro(CleanupPadInst) \ @@ -1592,7 +1599,10 @@ LLVMTypeRef LLVMX86MMXType(void); macro(ZExtInst) \ macro(ExtractValueInst) \ macro(LoadInst) \ - macro(VAArgInst) + macro(VAArgInst) \ + macro(AtomicCmpXchgInst) \ + macro(AtomicRMWInst) \ + macro(FenceInst) /** * @defgroup LLVMCCoreValueGeneral General APIs @@ -3807,8 +3817,12 @@ LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str, const char *Name); LLVMBool LLVMGetVolatile(LLVMValueRef MemoryAccessInst); void LLVMSetVolatile(LLVMValueRef MemoryAccessInst, LLVMBool IsVolatile); +LLVMBool LLVMGetWeak(LLVMValueRef CmpXchgInst); +void LLVMSetWeak(LLVMValueRef CmpXchgInst, LLVMBool IsWeak); LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemoryAccessInst); void LLVMSetOrdering(LLVMValueRef MemoryAccessInst, LLVMAtomicOrdering Ordering); +LLVMAtomicRMWBinOp LLVMGetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst); +void LLVMSetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst, LLVMAtomicRMWBinOp BinOp); /* Casts */ LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef, LLVMValueRef Val, diff --git a/include/llvm-c/DebugInfo.h b/include/llvm-c/DebugInfo.h index 33c8110a863c..41e9f96bbb92 100644 --- a/include/llvm-c/DebugInfo.h +++ b/include/llvm-c/DebugInfo.h @@ -32,7 +32,7 @@ typedef enum { LLVMDIFlagPublic = 3, LLVMDIFlagFwdDecl = 1 << 2, LLVMDIFlagAppleBlock = 1 << 3, - LLVMDIFlagBlockByrefStruct = 1 << 4, + LLVMDIFlagReservedBit4 = 1 << 4, LLVMDIFlagVirtual = 1 << 5, LLVMDIFlagArtificial = 1 << 6, LLVMDIFlagExplicit = 1 << 7, @@ -170,6 +170,19 @@ typedef unsigned LLVMMetadataKind; typedef unsigned LLVMDWARFTypeEncoding; /** + * Describes the kind of macro declaration used for LLVMDIBuilderCreateMacro. + * @see llvm::dwarf::MacinfoRecordType + * @note Values are from DW_MACINFO_* constants in the DWARF specification. + */ +typedef enum { + LLVMDWARFMacinfoRecordTypeDefine = 0x01, + LLVMDWARFMacinfoRecordTypeMacro = 0x02, + LLVMDWARFMacinfoRecordTypeStartFile = 0x03, + LLVMDWARFMacinfoRecordTypeEndFile = 0x04, + LLVMDWARFMacinfoRecordTypeVendorExt = 0xff +} LLVMDWARFMacinfoRecordType; + +/** * The current debug metadata version number. */ unsigned LLVMDebugMetadataVersion(void); @@ -522,6 +535,38 @@ LLVMDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder, LLVMDIFlags Flags); /** + * Create debugging information entry for a macro. + * @param Builder The DIBuilder. + * @param ParentMacroFile Macro parent (could be NULL). + * @param Line Source line number where the macro is defined. + * @param RecordType DW_MACINFO_define or DW_MACINFO_undef. + * @param Name Macro name. + * @param NameLen Macro name length. + * @param Value Macro value. + * @param ValueLen Macro value length. + */ +LLVMMetadataRef LLVMDIBuilderCreateMacro(LLVMDIBuilderRef Builder, + LLVMMetadataRef ParentMacroFile, + unsigned Line, + LLVMDWARFMacinfoRecordType RecordType, + const char *Name, size_t NameLen, + const char *Value, size_t ValueLen); + +/** + * Create debugging information temporary entry for a macro file. + * List of macro node direct children will be calculated by DIBuilder, + * using the \p ParentMacroFile relationship. + * @param Builder The DIBuilder. + * @param ParentMacroFile Macro parent (could be NULL). + * @param Line Source line number where the macro file is included. + * @param File File descriptor containing the name of the macro file. + */ +LLVMMetadataRef +LLVMDIBuilderCreateTempMacroFile(LLVMDIBuilderRef Builder, + LLVMMetadataRef ParentMacroFile, unsigned Line, + LLVMMetadataRef File); + +/** * Create debugging information entry for an enumerator. * @param Builder The DIBuilder. * @param Name Enumerator name. diff --git a/include/llvm-c/Remarks.h b/include/llvm-c/Remarks.h index 88eb5120c57c..5444aebddd60 100644 --- a/include/llvm-c/Remarks.h +++ b/include/llvm-c/Remarks.h @@ -30,7 +30,8 @@ extern "C" { * @{ */ -#define REMARKS_API_VERSION 0 +// 0 -> 1: Bitstream remarks support. +#define REMARKS_API_VERSION 1 /** * The type of the emitted remark. @@ -241,6 +242,20 @@ extern LLVMRemarkParserRef LLVMRemarkParserCreateYAML(const void *Buf, uint64_t Size); /** + * Creates a remark parser that can be used to parse the buffer located in \p + * Buf of size \p Size bytes. + * + * \p Buf cannot be `NULL`. + * + * This function should be paired with LLVMRemarkParserDispose() to avoid + * leaking resources. + * + * \since REMARKS_API_VERSION=1 + */ +extern LLVMRemarkParserRef LLVMRemarkParserCreateBitstream(const void *Buf, + uint64_t Size); + +/** * Returns the next remark in the file. * * The value pointed to by the return value needs to be disposed using a call to diff --git a/include/llvm-c/Transforms/IPO.h b/include/llvm-c/Transforms/IPO.h index 7a82ed464141..51d007581283 100644 --- a/include/llvm-c/Transforms/IPO.h +++ b/include/llvm-c/Transforms/IPO.h @@ -34,6 +34,9 @@ void LLVMAddArgumentPromotionPass(LLVMPassManagerRef PM); /** See llvm::createConstantMergePass function. */ void LLVMAddConstantMergePass(LLVMPassManagerRef PM); +/** See llvm::createMergeFunctionsPass function. */ +void LLVMAddMergeFunctionsPass(LLVMPassManagerRef PM); + /** See llvm::createCalledValuePropagationPass function. */ void LLVMAddCalledValuePropagationPass(LLVMPassManagerRef PM); @@ -67,6 +70,21 @@ void LLVMAddIPSCCPPass(LLVMPassManagerRef PM); /** See llvm::createInternalizePass function. */ void LLVMAddInternalizePass(LLVMPassManagerRef, unsigned AllButMain); +/** + * Create and add the internalize pass to the given pass manager with the + * provided preservation callback. + * + * The context parameter is forwarded to the callback on each invocation. + * As such, it is the responsibility of the caller to extend its lifetime + * until execution of this pass has finished. + * + * @see llvm::createInternalizePass function. + */ +void LLVMAddInternalizePassWithMustPreservePredicate( + LLVMPassManagerRef PM, + void *Context, + LLVMBool (*MustPreserve)(LLVMValueRef, void *)); + /** See llvm::createStripDeadPrototypesPass function. */ void LLVMAddStripDeadPrototypesPass(LLVMPassManagerRef PM); diff --git a/include/llvm-c/Transforms/Scalar.h b/include/llvm-c/Transforms/Scalar.h index 031cf98b2df2..6f3a3d8b3750 100644 --- a/include/llvm-c/Transforms/Scalar.h +++ b/include/llvm-c/Transforms/Scalar.h @@ -35,6 +35,9 @@ extern "C" { /** See llvm::createAggressiveDCEPass function. */ void LLVMAddAggressiveDCEPass(LLVMPassManagerRef PM); +/** See llvm::createDeadCodeEliminationPass function. */ +void LLVMAddDCEPass(LLVMPassManagerRef PM); + /** See llvm::createBitTrackingDCEPass function. */ void LLVMAddBitTrackingDCEPass(LLVMPassManagerRef PM); @@ -144,6 +147,9 @@ void LLVMAddEarlyCSEMemSSAPass(LLVMPassManagerRef PM); /** See llvm::createLowerExpectIntrinsicPass function */ void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM); +/** See llvm::createLowerConstantIntrinsicsPass function */ +void LLVMAddLowerConstantIntrinsicsPass(LLVMPassManagerRef PM); + /** See llvm::createTypeBasedAliasAnalysisPass function */ void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM); diff --git a/include/llvm-c/lto.h b/include/llvm-c/lto.h index 2467722b1954..41e6067cf44f 100644 --- a/include/llvm-c/lto.h +++ b/include/llvm-c/lto.h @@ -44,7 +44,7 @@ typedef bool lto_bool_t; * @{ */ -#define LTO_API_VERSION 24 +#define LTO_API_VERSION 25 /** * \since prior to LTO_API_VERSION=3 @@ -550,6 +550,56 @@ extern void lto_codegen_set_should_embed_uselists(lto_code_gen_t cg, lto_bool_t ShouldEmbedUselists); +/** Opaque reference to an LTO input file */ +typedef struct LLVMOpaqueLTOInput *lto_input_t; + +/** + * Creates an LTO input file from a buffer. The path + * argument is used for diagnotics as this function + * otherwise does not know which file the given buffer + * is associated with. + * + * \since LTO_API_VERSION=24 + */ +extern lto_input_t lto_input_create(const void *buffer, + size_t buffer_size, + const char *path); + +/** + * Frees all memory internally allocated by the LTO input file. + * Upon return the lto_module_t is no longer valid. + * + * \since LTO_API_VERSION=24 + */ +extern void lto_input_dispose(lto_input_t input); + +/** + * Returns the number of dependent library specifiers + * for the given LTO input file. + * + * \since LTO_API_VERSION=24 + */ +extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input); + +/** + * Returns the ith dependent library specifier + * for the given LTO input file. The returned + * string is not null-terminated. + * + * \since LTO_API_VERSION=24 + */ +extern const char * lto_input_get_dependent_library(lto_input_t input, + size_t index, + size_t *size); + +/** + * Returns the list of libcall symbols that can be generated by LTO + * that might not be visible from the symbol table of bitcode files. + * + * \since prior to LTO_API_VERSION=25 + */ +extern const char *const *lto_runtime_lib_symbols_list(size_t *size); + /** * @} // endgoup LLVMCLTO * @defgroup LLVMCTLTO ThinLTO @@ -846,48 +896,6 @@ thinlto_codegen_set_cache_size_megabytes(thinlto_code_gen_t cg, extern void thinlto_codegen_set_cache_size_files(thinlto_code_gen_t cg, unsigned max_size_files); -/** Opaque reference to an LTO input file */ -typedef struct LLVMOpaqueLTOInput *lto_input_t; - -/** - * Creates an LTO input file from a buffer. The path - * argument is used for diagnotics as this function - * otherwise does not know which file the given buffer - * is associated with. - * - * \since LTO_API_VERSION=24 - */ -extern lto_input_t lto_input_create(const void *buffer, - size_t buffer_size, - const char *path); - -/** - * Frees all memory internally allocated by the LTO input file. - * Upon return the lto_module_t is no longer valid. - * - * \since LTO_API_VERSION=24 - */ -extern void lto_input_dispose(lto_input_t input); - -/** - * Returns the number of dependent library specifiers - * for the given LTO input file. - * - * \since LTO_API_VERSION=24 - */ -extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input); - -/** - * Returns the ith dependent library specifier - * for the given LTO input file. The returned - * string is not null-terminated. - * - * \since LTO_API_VERSION=24 - */ -extern const char * lto_input_get_dependent_library(lto_input_t input, - size_t index, - size_t *size); - /** * @} // endgroup LLVMCTLTO_CACHING */ diff --git a/include/llvm/ADT/APFloat.h b/include/llvm/ADT/APFloat.h index a9648d35cf5d..1c4969733791 100644 --- a/include/llvm/ADT/APFloat.h +++ b/include/llvm/ADT/APFloat.h @@ -192,6 +192,11 @@ struct APFloatBase { /// IEEE-754R 7: Default exception handling. /// /// opUnderflow or opOverflow are always returned or-ed with opInexact. + /// + /// APFloat models this behavior specified by IEEE-754: + /// "For operations producing results in floating-point format, the default + /// result of an operation that signals the invalid operation exception + /// shall be a quiet NaN." enum opStatus { opOK = 0x00, opInvalidOp = 0x01, diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h index 2381b75e08b1..8dce5a621bb3 100644 --- a/include/llvm/ADT/APInt.h +++ b/include/llvm/ADT/APInt.h @@ -1467,6 +1467,13 @@ public: U.pVal[whichWord(BitPosition)] &= Mask; } + /// Set bottom loBits bits to 0. + void clearLowBits(unsigned loBits) { + assert(loBits <= BitWidth && "More bits than bitwidth"); + APInt Keep = getHighBitsSet(BitWidth, BitWidth - loBits); + *this &= Keep; + } + /// Set the sign bit to 0. void clearSignBit() { clearBit(BitWidth - 1); @@ -1496,9 +1503,11 @@ public: /// Insert the bits from a smaller APInt starting at bitPosition. void insertBits(const APInt &SubBits, unsigned bitPosition); + void insertBits(uint64_t SubBits, unsigned bitPosition, unsigned numBits); /// Return an APInt with the extracted bits [bitPosition,bitPosition+numBits). APInt extractBits(unsigned numBits, unsigned bitPosition) const; + uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const; /// @} /// \name Value Characterization Functions diff --git a/include/llvm/ADT/Any.h b/include/llvm/ADT/Any.h index 5dcd6e73c54f..49657e02a991 100644 --- a/include/llvm/ADT/Any.h +++ b/include/llvm/ADT/Any.h @@ -38,7 +38,7 @@ class Any { explicit StorageImpl(T &&Value) : Value(std::move(Value)) {} std::unique_ptr<StorageBase> clone() const override { - return llvm::make_unique<StorageImpl<T>>(Value); + return std::make_unique<StorageImpl<T>>(Value); } const void *id() const override { return &TypeId<T>::Id; } @@ -78,7 +78,7 @@ public: int>::type = 0> Any(T &&Value) { using U = typename std::decay<T>::type; - Storage = llvm::make_unique<StorageImpl<U>>(std::forward<T>(Value)); + Storage = std::make_unique<StorageImpl<U>>(std::forward<T>(Value)); } Any(Any &&Other) : Storage(std::move(Other.Storage)) {} diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h index 773c88f7c9f9..f6455d3fa412 100644 --- a/include/llvm/ADT/ArrayRef.h +++ b/include/llvm/ADT/ArrayRef.h @@ -481,6 +481,12 @@ namespace llvm { return Vec; } + /// Construct an ArrayRef from a std::array. + template <typename T, std::size_t N> + ArrayRef<T> makeArrayRef(const std::array<T, N> &Arr) { + return Arr; + } + /// Construct an ArrayRef from an ArrayRef (no-op) (const) template <typename T> ArrayRef<T> makeArrayRef(const ArrayRef<T> &Vec) { return Vec; diff --git a/include/llvm/ADT/DenseMap.h b/include/llvm/ADT/DenseMap.h index a05cf8130d3c..948a6e6bfb38 100644 --- a/include/llvm/ADT/DenseMap.h +++ b/include/llvm/ADT/DenseMap.h @@ -38,33 +38,7 @@ namespace detail { // implementation without requiring two members. template <typename KeyT, typename ValueT> struct DenseMapPair : public std::pair<KeyT, ValueT> { - - // FIXME: Switch to inheriting constructors when we drop support for older - // clang versions. - // NOTE: This default constructor is declared with '{}' rather than - // '= default' to work around a separate bug in clang-3.8. This can - // also go when we switch to inheriting constructors. - DenseMapPair() {} - - DenseMapPair(const KeyT &Key, const ValueT &Value) - : std::pair<KeyT, ValueT>(Key, Value) {} - - DenseMapPair(KeyT &&Key, ValueT &&Value) - : std::pair<KeyT, ValueT>(std::move(Key), std::move(Value)) {} - - template <typename AltKeyT, typename AltValueT> - DenseMapPair(AltKeyT &&AltKey, AltValueT &&AltValue, - typename std::enable_if< - std::is_convertible<AltKeyT, KeyT>::value && - std::is_convertible<AltValueT, ValueT>::value>::type * = 0) - : std::pair<KeyT, ValueT>(std::forward<AltKeyT>(AltKey), - std::forward<AltValueT>(AltValue)) {} - - template <typename AltPairT> - DenseMapPair(AltPairT &&AltPair, - typename std::enable_if<std::is_convertible< - AltPairT, std::pair<KeyT, ValueT>>::value>::type * = nullptr) - : std::pair<KeyT, ValueT>(std::forward<AltPairT>(AltPair)) {} + using std::pair<KeyT, ValueT>::pair; KeyT &getFirst() { return std::pair<KeyT, ValueT>::first; } const KeyT &getFirst() const { return std::pair<KeyT, ValueT>::first; } @@ -748,7 +722,7 @@ public: ~DenseMap() { this->destroyAll(); - operator delete(Buckets); + deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); } void swap(DenseMap& RHS) { @@ -768,7 +742,7 @@ public: DenseMap& operator=(DenseMap &&other) { this->destroyAll(); - operator delete(Buckets); + deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); init(0); swap(other); return *this; @@ -776,7 +750,7 @@ public: void copyFrom(const DenseMap& other) { this->destroyAll(); - operator delete(Buckets); + deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); if (allocateBuckets(other.NumBuckets)) { this->BaseT::copyFrom(other); } else { @@ -809,10 +783,12 @@ public: this->moveFromOldBuckets(OldBuckets, OldBuckets+OldNumBuckets); // Free the old table. - operator delete(OldBuckets); + deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets, + alignof(BucketT)); } void shrink_and_clear() { + unsigned OldNumBuckets = NumBuckets; unsigned OldNumEntries = NumEntries; this->destroyAll(); @@ -825,7 +801,8 @@ public: return; } - operator delete(Buckets); + deallocate_buffer(Buckets, sizeof(BucketT) * OldNumBuckets, + alignof(BucketT)); init(NewNumBuckets); } @@ -861,7 +838,8 @@ private: return false; } - Buckets = static_cast<BucketT*>(operator new(sizeof(BucketT) * NumBuckets)); + Buckets = static_cast<BucketT *>( + allocate_buffer(sizeof(BucketT) * NumBuckets, alignof(BucketT))); return true; } }; @@ -1076,7 +1054,8 @@ public: this->moveFromOldBuckets(OldRep.Buckets, OldRep.Buckets+OldRep.NumBuckets); // Free the old table. - operator delete(OldRep.Buckets); + deallocate_buffer(OldRep.Buckets, sizeof(BucketT) * OldRep.NumBuckets, + alignof(BucketT)); } void shrink_and_clear() { @@ -1160,15 +1139,17 @@ private: if (Small) return; - operator delete(getLargeRep()->Buckets); + deallocate_buffer(getLargeRep()->Buckets, + sizeof(BucketT) * getLargeRep()->NumBuckets, + alignof(BucketT)); getLargeRep()->~LargeRep(); } LargeRep allocateBuckets(unsigned Num) { assert(Num > InlineBuckets && "Must allocate more buckets than are inline"); - LargeRep Rep = { - static_cast<BucketT*>(operator new(sizeof(BucketT) * Num)), Num - }; + LargeRep Rep = {static_cast<BucketT *>(allocate_buffer( + sizeof(BucketT) * Num, alignof(BucketT))), + Num}; return Rep; } }; diff --git a/include/llvm/ADT/DenseMapInfo.h b/include/llvm/ADT/DenseMapInfo.h index 5ef6f3ad1b04..bd4c60c8f13e 100644 --- a/include/llvm/ADT/DenseMapInfo.h +++ b/include/llvm/ADT/DenseMapInfo.h @@ -17,7 +17,7 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/PointerLikeTypeTraits.h" -#include "llvm/Support/ScalableSize.h" +#include "llvm/Support/TypeSize.h" #include <cassert> #include <cstddef> #include <cstdint> @@ -67,6 +67,17 @@ template<> struct DenseMapInfo<char> { } }; +// Provide DenseMapInfo for unsigned chars. +template <> struct DenseMapInfo<unsigned char> { + static inline unsigned char getEmptyKey() { return ~0; } + static inline unsigned char getTombstoneKey() { return ~0 - 1; } + static unsigned getHashValue(const unsigned char &Val) { return Val * 37U; } + + static bool isEqual(const unsigned char &LHS, const unsigned char &RHS) { + return LHS == RHS; + } +}; + // Provide DenseMapInfo for unsigned shorts. template <> struct DenseMapInfo<unsigned short> { static inline unsigned short getEmptyKey() { return 0xFFFF; } diff --git a/include/llvm/ADT/DirectedGraph.h b/include/llvm/ADT/DirectedGraph.h new file mode 100644 index 000000000000..f6a358d99cd2 --- /dev/null +++ b/include/llvm/ADT/DirectedGraph.h @@ -0,0 +1,270 @@ +//===- llvm/ADT/DirectedGraph.h - Directed Graph ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the interface and a base class implementation for a +// directed graph. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_DIRECTEDGRAPH_H +#define LLVM_ADT_DIRECTEDGRAPH_H + +#include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +/// Represent an edge in the directed graph. +/// The edge contains the target node it connects to. +template <class NodeType, class EdgeType> class DGEdge { +public: + DGEdge() = delete; + /// Create an edge pointing to the given node \p N. + explicit DGEdge(NodeType &N) : TargetNode(N) {} + explicit DGEdge(const DGEdge<NodeType, EdgeType> &E) + : TargetNode(E.TargetNode) {} + DGEdge<NodeType, EdgeType> &operator=(const DGEdge<NodeType, EdgeType> &E) { + TargetNode = E.TargetNode; + return *this; + } + + /// Static polymorphism: delegate implementation (via isEqualTo) to the + /// derived class. + bool operator==(const EdgeType &E) const { return getDerived().isEqualTo(E); } + bool operator!=(const EdgeType &E) const { return !operator==(E); } + + /// Retrieve the target node this edge connects to. + const NodeType &getTargetNode() const { return TargetNode; } + NodeType &getTargetNode() { + return const_cast<NodeType &>( + static_cast<const DGEdge<NodeType, EdgeType> &>(*this).getTargetNode()); + } + +protected: + // As the default implementation use address comparison for equality. + bool isEqualTo(const EdgeType &E) const { return this == &E; } + + // Cast the 'this' pointer to the derived type and return a reference. + EdgeType &getDerived() { return *static_cast<EdgeType *>(this); } + const EdgeType &getDerived() const { + return *static_cast<const EdgeType *>(this); + } + + // The target node this edge connects to. + NodeType &TargetNode; +}; + +/// Represent a node in the directed graph. +/// The node has a (possibly empty) list of outgoing edges. +template <class NodeType, class EdgeType> class DGNode { +public: + using EdgeListTy = SetVector<EdgeType *>; + using iterator = typename EdgeListTy::iterator; + using const_iterator = typename EdgeListTy::const_iterator; + + /// Create a node with a single outgoing edge \p E. + explicit DGNode(EdgeType &E) : Edges() { Edges.insert(&E); } + DGNode() = default; + + explicit DGNode(const DGNode<NodeType, EdgeType> &N) : Edges(N.Edges) {} + DGNode(DGNode<NodeType, EdgeType> &&N) : Edges(std::move(N.Edges)) {} + + DGNode<NodeType, EdgeType> &operator=(const DGNode<NodeType, EdgeType> &N) { + Edges = N.Edges; + return *this; + } + DGNode<NodeType, EdgeType> &operator=(const DGNode<NodeType, EdgeType> &&N) { + Edges = std::move(N.Edges); + return *this; + } + + /// Static polymorphism: delegate implementation (via isEqualTo) to the + /// derived class. + bool operator==(const NodeType &N) const { return getDerived().isEqualTo(N); } + bool operator!=(const NodeType &N) const { return !operator==(N); } + + const_iterator begin() const { return Edges.begin(); } + const_iterator end() const { return Edges.end(); } + iterator begin() { return Edges.begin(); } + iterator end() { return Edges.end(); } + const EdgeType &front() const { return *Edges.front(); } + EdgeType &front() { return *Edges.front(); } + const EdgeType &back() const { return *Edges.back(); } + EdgeType &back() { return *Edges.back(); } + + /// Collect in \p EL, all the edges from this node to \p N. + /// Return true if at least one edge was found, and false otherwise. + /// Note that this implementation allows more than one edge to connect + /// a given pair of nodes. + bool findEdgesTo(const NodeType &N, SmallVectorImpl<EdgeType *> &EL) const { + assert(EL.empty() && "Expected the list of edges to be empty."); + for (auto *E : Edges) + if (E->getTargetNode() == N) + EL.push_back(E); + return !EL.empty(); + } + + /// Add the given edge \p E to this node, if it doesn't exist already. Returns + /// true if the edge is added and false otherwise. + bool addEdge(EdgeType &E) { return Edges.insert(&E); } + + /// Remove the given edge \p E from this node, if it exists. + void removeEdge(EdgeType &E) { Edges.remove(&E); } + + /// Test whether there is an edge that goes from this node to \p N. + bool hasEdgeTo(const NodeType &N) const { + return (findEdgeTo(N) != Edges.end()); + } + + /// Retrieve the outgoing edges for the node. + const EdgeListTy &getEdges() const { return Edges; } + EdgeListTy &getEdges() { + return const_cast<EdgeListTy &>( + static_cast<const DGNode<NodeType, EdgeType> &>(*this).Edges); + } + + /// Clear the outgoing edges. + void clear() { Edges.clear(); } + +protected: + // As the default implementation use address comparison for equality. + bool isEqualTo(const NodeType &N) const { return this == &N; } + + // Cast the 'this' pointer to the derived type and return a reference. + NodeType &getDerived() { return *static_cast<NodeType *>(this); } + const NodeType &getDerived() const { + return *static_cast<const NodeType *>(this); + } + + /// Find an edge to \p N. If more than one edge exists, this will return + /// the first one in the list of edges. + const_iterator findEdgeTo(const NodeType &N) const { + return llvm::find_if( + Edges, [&N](const EdgeType *E) { return E->getTargetNode() == N; }); + } + + // The list of outgoing edges. + EdgeListTy Edges; +}; + +/// Directed graph +/// +/// The graph is represented by a table of nodes. +/// Each node contains a (possibly empty) list of outgoing edges. +/// Each edge contains the target node it connects to. +template <class NodeType, class EdgeType> class DirectedGraph { +protected: + using NodeListTy = SmallVector<NodeType *, 10>; + using EdgeListTy = SmallVector<EdgeType *, 10>; +public: + using iterator = typename NodeListTy::iterator; + using const_iterator = typename NodeListTy::const_iterator; + using DGraphType = DirectedGraph<NodeType, EdgeType>; + + DirectedGraph() = default; + explicit DirectedGraph(NodeType &N) : Nodes() { addNode(N); } + DirectedGraph(const DGraphType &G) : Nodes(G.Nodes) {} + DirectedGraph(DGraphType &&RHS) : Nodes(std::move(RHS.Nodes)) {} + DGraphType &operator=(const DGraphType &G) { + Nodes = G.Nodes; + return *this; + } + DGraphType &operator=(const DGraphType &&G) { + Nodes = std::move(G.Nodes); + return *this; + } + + const_iterator begin() const { return Nodes.begin(); } + const_iterator end() const { return Nodes.end(); } + iterator begin() { return Nodes.begin(); } + iterator end() { return Nodes.end(); } + const NodeType &front() const { return *Nodes.front(); } + NodeType &front() { return *Nodes.front(); } + const NodeType &back() const { return *Nodes.back(); } + NodeType &back() { return *Nodes.back(); } + + size_t size() const { return Nodes.size(); } + + /// Find the given node \p N in the table. + const_iterator findNode(const NodeType &N) const { + return llvm::find_if(Nodes, + [&N](const NodeType *Node) { return *Node == N; }); + } + iterator findNode(const NodeType &N) { + return const_cast<iterator>( + static_cast<const DGraphType &>(*this).findNode(N)); + } + + /// Add the given node \p N to the graph if it is not already present. + bool addNode(NodeType &N) { + if (findNode(N) != Nodes.end()) + return false; + Nodes.push_back(&N); + return true; + } + + /// Collect in \p EL all edges that are coming into node \p N. Return true + /// if at least one edge was found, and false otherwise. + bool findIncomingEdgesToNode(const NodeType &N, SmallVectorImpl<EdgeType*> &EL) const { + assert(EL.empty() && "Expected the list of edges to be empty."); + EdgeListTy TempList; + for (auto *Node : Nodes) { + if (*Node == N) + continue; + Node->findEdgesTo(N, TempList); + EL.insert(EL.end(), TempList.begin(), TempList.end()); + TempList.clear(); + } + return !EL.empty(); + } + + /// Remove the given node \p N from the graph. If the node has incoming or + /// outgoing edges, they are also removed. Return true if the node was found + /// and then removed, and false if the node was not found in the graph to + /// begin with. + bool removeNode(NodeType &N) { + iterator IT = findNode(N); + if (IT == Nodes.end()) + return false; + // Remove incoming edges. + EdgeListTy EL; + for (auto *Node : Nodes) { + if (*Node == N) + continue; + Node->findEdgesTo(N, EL); + for (auto *E : EL) + Node->removeEdge(*E); + EL.clear(); + } + N.clear(); + Nodes.erase(IT); + return true; + } + + /// Assuming nodes \p Src and \p Dst are already in the graph, connect node \p + /// Src to node \p Dst using the provided edge \p E. Return true if \p Src is + /// not already connected to \p Dst via \p E, and false otherwise. + bool connect(NodeType &Src, NodeType &Dst, EdgeType &E) { + assert(findNode(Src) != Nodes.end() && "Src node should be present."); + assert(findNode(Dst) != Nodes.end() && "Dst node should be present."); + assert((E.getTargetNode() == Dst) && + "Target of the given edge does not match Dst."); + return Src.addEdge(E); + } + +protected: + // The list of nodes in the graph. + NodeListTy Nodes; +}; + +} // namespace llvm + +#endif // LLVM_ADT_DIRECTEDGRAPH_H diff --git a/include/llvm/ADT/Hashing.h b/include/llvm/ADT/Hashing.h index 008188bfa210..b22606bdb518 100644 --- a/include/llvm/ADT/Hashing.h +++ b/include/llvm/ADT/Hashing.h @@ -45,7 +45,6 @@ #define LLVM_ADT_HASHING_H #include "llvm/Support/DataTypes.h" -#include "llvm/Support/Host.h" #include "llvm/Support/SwapByteOrder.h" #include "llvm/Support/type_traits.h" #include <algorithm> diff --git a/include/llvm/ADT/IntervalMap.h b/include/llvm/ADT/IntervalMap.h index 12828c4cfdab..a02876ee77f3 100644 --- a/include/llvm/ADT/IntervalMap.h +++ b/include/llvm/ADT/IntervalMap.h @@ -963,8 +963,8 @@ public: private: // The root data is either a RootLeaf or a RootBranchData instance. - LLVM_ALIGNAS(RootLeaf) LLVM_ALIGNAS(RootBranchData) - AlignedCharArrayUnion<RootLeaf, RootBranchData> data; + alignas(RootLeaf) alignas(RootBranchData) + AlignedCharArrayUnion<RootLeaf, RootBranchData> data; // Tree height. // 0: Leaves in root. diff --git a/include/llvm/ADT/PointerIntPair.h b/include/llvm/ADT/PointerIntPair.h index 24a2bb67a36e..fa6bf1504469 100644 --- a/include/llvm/ADT/PointerIntPair.h +++ b/include/llvm/ADT/PointerIntPair.h @@ -13,6 +13,7 @@ #ifndef LLVM_ADT_POINTERINTPAIR_H #define LLVM_ADT_POINTERINTPAIR_H +#include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include "llvm/Support/type_traits.h" #include <cassert> @@ -59,19 +60,19 @@ public: IntType getInt() const { return (IntType)Info::getInt(Value); } - void setPointer(PointerTy PtrVal) { + void setPointer(PointerTy PtrVal) LLVM_LVALUE_FUNCTION { Value = Info::updatePointer(Value, PtrVal); } - void setInt(IntType IntVal) { + void setInt(IntType IntVal) LLVM_LVALUE_FUNCTION { Value = Info::updateInt(Value, static_cast<intptr_t>(IntVal)); } - void initWithPointer(PointerTy PtrVal) { + void initWithPointer(PointerTy PtrVal) LLVM_LVALUE_FUNCTION { Value = Info::updatePointer(0, PtrVal); } - void setPointerAndInt(PointerTy PtrVal, IntType IntVal) { + void setPointerAndInt(PointerTy PtrVal, IntType IntVal) LLVM_LVALUE_FUNCTION { Value = Info::updateInt(Info::updatePointer(0, PtrVal), static_cast<intptr_t>(IntVal)); } @@ -89,7 +90,7 @@ public: void *getOpaqueValue() const { return reinterpret_cast<void *>(Value); } - void setFromOpaqueValue(void *Val) { + void setFromOpaqueValue(void *Val) LLVM_LVALUE_FUNCTION { Value = reinterpret_cast<intptr_t>(Val); } diff --git a/include/llvm/ADT/PointerUnion.h b/include/llvm/ADT/PointerUnion.h index 2bcdf546c6e4..98c905775a77 100644 --- a/include/llvm/ADT/PointerUnion.h +++ b/include/llvm/ADT/PointerUnion.h @@ -54,21 +54,14 @@ struct PointerUnionTypeSelectorReturn< }; namespace pointer_union_detail { - constexpr int constexprMin(int a, int b) { return a < b ? a : b; } /// Determine the number of bits required to store integers with values < n. /// This is ceil(log2(n)). constexpr int bitsRequired(unsigned n) { return n > 1 ? 1 + bitsRequired((n + 1) / 2) : 0; } - // FIXME: In C++14, replace this with - // std::min({PointerLikeTypeTraits<Ts>::NumLowBitsAvailable...}) - template <typename T> constexpr int lowBitsAvailable() { - return PointerLikeTypeTraits<T>::NumLowBitsAvailable; - } - template <typename T1, typename T2, typename... Ts> - constexpr int lowBitsAvailable() { - return constexprMin(lowBitsAvailable<T1>(), lowBitsAvailable<T2, Ts...>()); + template <typename... Ts> constexpr int lowBitsAvailable() { + return std::min<int>({PointerLikeTypeTraits<Ts>::NumLowBitsAvailable...}); } /// Find the index of a type in a list of types. TypeIndex<T, Us...>::Index @@ -167,10 +160,11 @@ class PointerUnion void *, pointer_union_detail::bitsRequired(sizeof...(PTs)), int, pointer_union_detail::PointerUnionUIntTraits<PTs...>>, 0, PTs...> { - // The first type is special in some ways, but we don't want PointerUnion to - // be a 'template <typename First, typename ...Rest>' because it's much more - // convenient to have a name for the whole pack. So split off the first type - // here. + // The first type is special because we want to directly cast a pointer to a + // default-initialized union to a pointer to the first type. But we don't + // want PointerUnion to be a 'template <typename First, typename ...Rest>' + // because it's much more convenient to have a name for the whole pack. So + // split off the first type here. using First = typename pointer_union_detail::GetFirstType<PTs...>::type; using Base = typename PointerUnion::PointerUnionMembers; @@ -182,12 +176,7 @@ public: /// Test if the pointer held in the union is null, regardless of /// which type it is. - bool isNull() const { - // Convert from the void* to one of the pointer types, to make sure that - // we recursively strip off low bits if we have a nested PointerUnion. - return !PointerLikeTypeTraits<First>::getFromVoidPointer( - this->Val.getPointer()); - } + bool isNull() const { return !this->Val.getPointer(); } explicit operator bool() const { return !isNull(); } @@ -226,7 +215,8 @@ public: First *getAddrOfPtr1() { assert(is<First>() && "Val is not the first pointer"); assert( - get<First>() == this->Val.getPointer() && + PointerLikeTypeTraits<First>::getAsVoidPointer(get<First>()) == + this->Val.getPointer() && "Can't get the address because PointerLikeTypeTraits changes the ptr"); return const_cast<First *>( reinterpret_cast<const First *>(this->Val.getAddrOfPointer())); diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h index 81dce0168c79..274933bc5204 100644 --- a/include/llvm/ADT/STLExtras.h +++ b/include/llvm/ADT/STLExtras.h @@ -95,18 +95,6 @@ template <class Ty> struct identity { } }; -template <class Ty> struct less_ptr { - bool operator()(const Ty* left, const Ty* right) const { - return *left < *right; - } -}; - -template <class Ty> struct greater_ptr { - bool operator()(const Ty* left, const Ty* right) const { - return *right < *left; - } -}; - /// An efficient, type-erasing, non-owning reference to a callable. This is /// intended for use as the type of a function parameter that is not used /// after the function in question returns. @@ -530,10 +518,6 @@ bool all_of(R &&range, UnaryPredicate P); template <typename R, typename UnaryPredicate> bool any_of(R &&range, UnaryPredicate P); -template <size_t... I> struct index_sequence; - -template <class... Ts> struct index_sequence_for; - namespace detail { using std::declval; @@ -568,38 +552,38 @@ struct zip_common : public zip_traits<ZipType, Iters...> { std::tuple<Iters...> iterators; protected: - template <size_t... Ns> value_type deref(index_sequence<Ns...>) const { + template <size_t... Ns> value_type deref(std::index_sequence<Ns...>) const { return value_type(*std::get<Ns>(iterators)...); } template <size_t... Ns> - decltype(iterators) tup_inc(index_sequence<Ns...>) const { + decltype(iterators) tup_inc(std::index_sequence<Ns...>) const { return std::tuple<Iters...>(std::next(std::get<Ns>(iterators))...); } template <size_t... Ns> - decltype(iterators) tup_dec(index_sequence<Ns...>) const { + decltype(iterators) tup_dec(std::index_sequence<Ns...>) const { return std::tuple<Iters...>(std::prev(std::get<Ns>(iterators))...); } public: zip_common(Iters &&... ts) : iterators(std::forward<Iters>(ts)...) {} - value_type operator*() { return deref(index_sequence_for<Iters...>{}); } + value_type operator*() { return deref(std::index_sequence_for<Iters...>{}); } const value_type operator*() const { - return deref(index_sequence_for<Iters...>{}); + return deref(std::index_sequence_for<Iters...>{}); } ZipType &operator++() { - iterators = tup_inc(index_sequence_for<Iters...>{}); + iterators = tup_inc(std::index_sequence_for<Iters...>{}); return *reinterpret_cast<ZipType *>(this); } ZipType &operator--() { static_assert(Base::IsBidirectional, "All inner iterators must be at least bidirectional."); - iterators = tup_dec(index_sequence_for<Iters...>{}); + iterators = tup_dec(std::index_sequence_for<Iters...>{}); return *reinterpret_cast<ZipType *>(this); } }; @@ -618,7 +602,8 @@ struct zip_first : public zip_common<zip_first<Iters...>, Iters...> { template <typename... Iters> class zip_shortest : public zip_common<zip_shortest<Iters...>, Iters...> { template <size_t... Ns> - bool test(const zip_shortest<Iters...> &other, index_sequence<Ns...>) const { + bool test(const zip_shortest<Iters...> &other, + std::index_sequence<Ns...>) const { return all_of(std::initializer_list<bool>{std::get<Ns>(this->iterators) != std::get<Ns>(other.iterators)...}, identity<bool>{}); @@ -630,7 +615,7 @@ public: zip_shortest(Iters &&... ts) : Base(std::forward<Iters>(ts)...) {} bool operator==(const zip_shortest<Iters...> &other) const { - return !test(other, index_sequence_for<Iters...>{}); + return !test(other, std::index_sequence_for<Iters...>{}); } }; @@ -646,18 +631,21 @@ public: private: std::tuple<Args...> ts; - template <size_t... Ns> iterator begin_impl(index_sequence<Ns...>) const { + template <size_t... Ns> + iterator begin_impl(std::index_sequence<Ns...>) const { return iterator(std::begin(std::get<Ns>(ts))...); } - template <size_t... Ns> iterator end_impl(index_sequence<Ns...>) const { + template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) const { return iterator(std::end(std::get<Ns>(ts))...); } public: zippy(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} - iterator begin() const { return begin_impl(index_sequence_for<Args...>{}); } - iterator end() const { return end_impl(index_sequence_for<Args...>{}); } + iterator begin() const { + return begin_impl(std::index_sequence_for<Args...>{}); + } + iterator end() const { return end_impl(std::index_sequence_for<Args...>{}); } }; } // end namespace detail @@ -727,20 +715,20 @@ private: template <size_t... Ns> bool test(const zip_longest_iterator<Iters...> &other, - index_sequence<Ns...>) const { + std::index_sequence<Ns...>) const { return llvm::any_of( std::initializer_list<bool>{std::get<Ns>(this->iterators) != std::get<Ns>(other.iterators)...}, identity<bool>{}); } - template <size_t... Ns> value_type deref(index_sequence<Ns...>) const { + template <size_t... Ns> value_type deref(std::index_sequence<Ns...>) const { return value_type( deref_or_none(std::get<Ns>(iterators), std::get<Ns>(end_iterators))...); } template <size_t... Ns> - decltype(iterators) tup_inc(index_sequence<Ns...>) const { + decltype(iterators) tup_inc(std::index_sequence<Ns...>) const { return std::tuple<Iters...>( next_or_end(std::get<Ns>(iterators), std::get<Ns>(end_iterators))...); } @@ -750,17 +738,19 @@ public: : iterators(std::forward<Iters>(ts.first)...), end_iterators(std::forward<Iters>(ts.second)...) {} - value_type operator*() { return deref(index_sequence_for<Iters...>{}); } + value_type operator*() { return deref(std::index_sequence_for<Iters...>{}); } - value_type operator*() const { return deref(index_sequence_for<Iters...>{}); } + value_type operator*() const { + return deref(std::index_sequence_for<Iters...>{}); + } zip_longest_iterator<Iters...> &operator++() { - iterators = tup_inc(index_sequence_for<Iters...>{}); + iterators = tup_inc(std::index_sequence_for<Iters...>{}); return *this; } bool operator==(const zip_longest_iterator<Iters...> &other) const { - return !test(other, index_sequence_for<Iters...>{}); + return !test(other, std::index_sequence_for<Iters...>{}); } }; @@ -777,12 +767,13 @@ public: private: std::tuple<Args...> ts; - template <size_t... Ns> iterator begin_impl(index_sequence<Ns...>) const { + template <size_t... Ns> + iterator begin_impl(std::index_sequence<Ns...>) const { return iterator(std::make_pair(adl_begin(std::get<Ns>(ts)), adl_end(std::get<Ns>(ts)))...); } - template <size_t... Ns> iterator end_impl(index_sequence<Ns...>) const { + template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) const { return iterator(std::make_pair(adl_end(std::get<Ns>(ts)), adl_end(std::get<Ns>(ts)))...); } @@ -790,8 +781,10 @@ private: public: zip_longest_range(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} - iterator begin() const { return begin_impl(index_sequence_for<Args...>{}); } - iterator end() const { return end_impl(index_sequence_for<Args...>{}); } + iterator begin() const { + return begin_impl(std::index_sequence_for<Args...>{}); + } + iterator end() const { return end_impl(std::index_sequence_for<Args...>{}); } }; } // namespace detail @@ -847,7 +840,7 @@ class concat_iterator /// Increments the first non-end iterator. /// /// It is an error to call this with all iterators at the end. - template <size_t... Ns> void increment(index_sequence<Ns...>) { + template <size_t... Ns> void increment(std::index_sequence<Ns...>) { // Build a sequence of functions to increment each iterator if possible. bool (concat_iterator::*IncrementHelperFns[])() = { &concat_iterator::incrementHelper<Ns>...}; @@ -876,7 +869,7 @@ class concat_iterator /// reference. /// /// It is an error to call this with all iterators at the end. - template <size_t... Ns> ValueT &get(index_sequence<Ns...>) const { + template <size_t... Ns> ValueT &get(std::index_sequence<Ns...>) const { // Build a sequence of functions to get from iterator if possible. ValueT *(concat_iterator::*GetHelperFns[])() const = { &concat_iterator::getHelper<Ns>...}; @@ -901,11 +894,13 @@ public: using BaseT::operator++; concat_iterator &operator++() { - increment(index_sequence_for<IterTs...>()); + increment(std::index_sequence_for<IterTs...>()); return *this; } - ValueT &operator*() const { return get(index_sequence_for<IterTs...>()); } + ValueT &operator*() const { + return get(std::index_sequence_for<IterTs...>()); + } bool operator==(const concat_iterator &RHS) const { return Begins == RHS.Begins && Ends == RHS.Ends; @@ -928,10 +923,10 @@ public: private: std::tuple<RangeTs...> Ranges; - template <size_t... Ns> iterator begin_impl(index_sequence<Ns...>) { + template <size_t... Ns> iterator begin_impl(std::index_sequence<Ns...>) { return iterator(std::get<Ns>(Ranges)...); } - template <size_t... Ns> iterator end_impl(index_sequence<Ns...>) { + template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) { return iterator(make_range(std::end(std::get<Ns>(Ranges)), std::end(std::get<Ns>(Ranges)))...); } @@ -940,8 +935,8 @@ public: concat_range(RangeTs &&... Ranges) : Ranges(std::forward<RangeTs>(Ranges)...) {} - iterator begin() { return begin_impl(index_sequence_for<RangeTs...>{}); } - iterator end() { return end_impl(index_sequence_for<RangeTs...>{}); } + iterator begin() { return begin_impl(std::index_sequence_for<RangeTs...>{}); } + iterator end() { return end_impl(std::index_sequence_for<RangeTs...>{}); } }; } // end namespace detail @@ -990,28 +985,6 @@ struct on_first { } }; -// A subset of N3658. More stuff can be added as-needed. - -/// Represents a compile-time sequence of integers. -template <class T, T... I> struct integer_sequence { - using value_type = T; - - static constexpr size_t size() { return sizeof...(I); } -}; - -/// Alias for the common case of a sequence of size_ts. -template <size_t... I> -struct index_sequence : integer_sequence<std::size_t, I...> {}; - -template <std::size_t N, std::size_t... I> -struct build_index_impl : build_index_impl<N - 1, N - 1, I...> {}; -template <std::size_t... I> -struct build_index_impl<0, I...> : index_sequence<I...> {}; - -/// Creates a compile-time integer sequence for a parameter pack. -template <class... Ts> -struct index_sequence_for : build_index_impl<sizeof...(Ts)> {}; - /// Utility type to build an inheritance chain that makes it easy to rank /// overload candidates. template <int N> struct rank : rank<N - 1> {}; @@ -1391,41 +1364,6 @@ void replace(Container &Cont, typename Container::iterator ContIt, // Extra additions to <memory> //===----------------------------------------------------------------------===// -// Implement make_unique according to N3656. - -/// Constructs a `new T()` with the given args and returns a -/// `unique_ptr<T>` which owns the object. -/// -/// Example: -/// -/// auto p = make_unique<int>(); -/// auto p = make_unique<std::tuple<int, int>>(0, 1); -template <class T, class... Args> -typename std::enable_if<!std::is_array<T>::value, std::unique_ptr<T>>::type -make_unique(Args &&... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); -} - -/// Constructs a `new T[n]` with the given args and returns a -/// `unique_ptr<T[]>` which owns the object. -/// -/// \param n size of the new array. -/// -/// Example: -/// -/// auto p = make_unique<int[]>(2); // value-initializes the array with 0's. -template <class T> -typename std::enable_if<std::is_array<T>::value && std::extent<T>::value == 0, - std::unique_ptr<T>>::type -make_unique(size_t n) { - return std::unique_ptr<T>(new typename std::remove_extent<T>::type[n]()); -} - -/// This function isn't used and is only here to provide better compile errors. -template <class T, class... Args> -typename std::enable_if<std::extent<T>::value != 0>::type -make_unique(Args &&...) = delete; - struct FreeDeleter { void operator()(void* v) { ::free(v); @@ -1439,20 +1377,6 @@ struct pair_hash { } }; -/// A functor like C++14's std::less<void> in its absence. -struct less { - template <typename A, typename B> bool operator()(A &&a, B &&b) const { - return std::forward<A>(a) < std::forward<B>(b); - } -}; - -/// A functor like C++14's std::equal<void> in its absence. -struct equal { - template <typename A, typename B> bool operator()(A &&a, B &&b) const { - return std::forward<A>(a) == std::forward<B>(b); - } -}; - /// Binary functor that adapts to any other binary functor after dereferencing /// operands. template <typename T> struct deref { @@ -1580,7 +1504,7 @@ template <typename R> detail::enumerator<R> enumerate(R &&TheRange) { namespace detail { template <typename F, typename Tuple, std::size_t... I> -auto apply_tuple_impl(F &&f, Tuple &&t, index_sequence<I...>) +auto apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) -> decltype(std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...)) { return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...); } @@ -1593,9 +1517,9 @@ auto apply_tuple_impl(F &&f, Tuple &&t, index_sequence<I...>) template <typename F, typename Tuple> auto apply_tuple(F &&f, Tuple &&t) -> decltype(detail::apply_tuple_impl( std::forward<F>(f), std::forward<Tuple>(t), - build_index_impl< + std::make_index_sequence< std::tuple_size<typename std::decay<Tuple>::type>::value>{})) { - using Indices = build_index_impl< + using Indices = std::make_index_sequence< std::tuple_size<typename std::decay<Tuple>::type>::value>; return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t), diff --git a/include/llvm/ADT/SmallBitVector.h b/include/llvm/ADT/SmallBitVector.h index 742450e6a951..61375c008022 100644 --- a/include/llvm/ADT/SmallBitVector.h +++ b/include/llvm/ADT/SmallBitVector.h @@ -290,7 +290,7 @@ public: ++Prev; uintptr_t Bits = getSmallBits(); // Mask in previous bits. - uintptr_t Mask = (1 << Prev) - 1; + uintptr_t Mask = (uintptr_t(1) << Prev) - 1; Bits |= Mask; if (Bits == ~uintptr_t(0) || Prev + 1 >= getSmallSize()) diff --git a/include/llvm/ADT/Statistic.h b/include/llvm/ADT/Statistic.h index 2ac59da596ef..b7387ddcf1c7 100644 --- a/include/llvm/ADT/Statistic.h +++ b/include/llvm/ADT/Statistic.h @@ -44,38 +44,39 @@ class raw_ostream; class raw_fd_ostream; class StringRef; -class Statistic { +class StatisticBase { public: const char *DebugType; const char *Name; const char *Desc; - std::atomic<unsigned> Value; - std::atomic<bool> Initialized; - unsigned getValue() const { return Value.load(std::memory_order_relaxed); } + StatisticBase(const char *DebugType, const char *Name, const char *Desc) + : DebugType(DebugType), Name(Name), Desc(Desc) {} + const char *getDebugType() const { return DebugType; } const char *getName() const { return Name; } const char *getDesc() const { return Desc; } +}; - /// construct - This should only be called for non-global statistics. - void construct(const char *debugtype, const char *name, const char *desc) { - DebugType = debugtype; - Name = name; - Desc = desc; - Value = 0; - Initialized = false; - } +class TrackingStatistic : public StatisticBase { +public: + std::atomic<unsigned> Value; + std::atomic<bool> Initialized; + + TrackingStatistic(const char *DebugType, const char *Name, const char *Desc) + : StatisticBase(DebugType, Name, Desc), Value(0), Initialized(false) {} + + unsigned getValue() const { return Value.load(std::memory_order_relaxed); } // Allow use of this class as the value itself. operator unsigned() const { return getValue(); } -#if LLVM_ENABLE_STATS - const Statistic &operator=(unsigned Val) { + const TrackingStatistic &operator=(unsigned Val) { Value.store(Val, std::memory_order_relaxed); return init(); } - const Statistic &operator++() { + const TrackingStatistic &operator++() { Value.fetch_add(1, std::memory_order_relaxed); return init(); } @@ -85,7 +86,7 @@ public: return Value.fetch_add(1, std::memory_order_relaxed); } - const Statistic &operator--() { + const TrackingStatistic &operator--() { Value.fetch_sub(1, std::memory_order_relaxed); return init(); } @@ -95,14 +96,14 @@ public: return Value.fetch_sub(1, std::memory_order_relaxed); } - const Statistic &operator+=(unsigned V) { + const TrackingStatistic &operator+=(unsigned V) { if (V == 0) return *this; Value.fetch_add(V, std::memory_order_relaxed); return init(); } - const Statistic &operator-=(unsigned V) { + const TrackingStatistic &operator-=(unsigned V) { if (V == 0) return *this; Value.fetch_sub(V, std::memory_order_relaxed); @@ -119,54 +120,57 @@ public: init(); } -#else // Statistics are disabled in release builds. - - const Statistic &operator=(unsigned Val) { +protected: + TrackingStatistic &init() { + if (!Initialized.load(std::memory_order_acquire)) + RegisterStatistic(); return *this; } - const Statistic &operator++() { - return *this; - } + void RegisterStatistic(); +}; - unsigned operator++(int) { - return 0; - } +class NoopStatistic : public StatisticBase { +public: + using StatisticBase::StatisticBase; - const Statistic &operator--() { - return *this; - } + unsigned getValue() const { return 0; } - unsigned operator--(int) { - return 0; - } + // Allow use of this class as the value itself. + operator unsigned() const { return 0; } - const Statistic &operator+=(const unsigned &V) { - return *this; - } + const NoopStatistic &operator=(unsigned Val) { return *this; } - const Statistic &operator-=(const unsigned &V) { - return *this; - } + const NoopStatistic &operator++() { return *this; } - void updateMax(unsigned V) {} + unsigned operator++(int) { return 0; } -#endif // LLVM_ENABLE_STATS + const NoopStatistic &operator--() { return *this; } -protected: - Statistic &init() { - if (!Initialized.load(std::memory_order_acquire)) - RegisterStatistic(); - return *this; - } + unsigned operator--(int) { return 0; } - void RegisterStatistic(); + const NoopStatistic &operator+=(const unsigned &V) { return *this; } + + const NoopStatistic &operator-=(const unsigned &V) { return *this; } + + void updateMax(unsigned V) {} }; +#if LLVM_ENABLE_STATS +using Statistic = TrackingStatistic; +#else +using Statistic = NoopStatistic; +#endif + // STATISTIC - A macro to make definition of statistics really simple. This // automatically passes the DEBUG_TYPE of the file into the statistic. #define STATISTIC(VARNAME, DESC) \ - static llvm::Statistic VARNAME = {DEBUG_TYPE, #VARNAME, DESC, {0}, {false}} + static llvm::Statistic VARNAME = {DEBUG_TYPE, #VARNAME, DESC} + +// ALWAYS_ENABLED_STATISTIC - A macro to define a statistic like STATISTIC but +// it is enabled even if LLVM_ENABLE_STATS is off. +#define ALWAYS_ENABLED_STATISTIC(VARNAME, DESC) \ + static llvm::TrackingStatistic VARNAME = {DEBUG_TYPE, #VARNAME, DESC} /// Enable the collection and printing of statistics. void EnableStatistics(bool PrintOnExit = true); diff --git a/include/llvm/ADT/StringExtras.h b/include/llvm/ADT/StringExtras.h index 16ac90bd6c89..ef1a11e0619b 100644 --- a/include/llvm/ADT/StringExtras.h +++ b/include/llvm/ADT/StringExtras.h @@ -345,7 +345,7 @@ inline void join_items_impl(std::string &Result, Sep Separator, const Arg1 &A1, join_items_impl(Result, Separator, std::forward<Args>(Items)...); } -inline size_t join_one_item_size(char C) { return 1; } +inline size_t join_one_item_size(char) { return 1; } inline size_t join_one_item_size(const char *S) { return S ? ::strlen(S) : 0; } template <typename T> inline size_t join_one_item_size(const T &Str) { diff --git a/include/llvm/ADT/StringMap.h b/include/llvm/ADT/StringMap.h index 8a586fc26709..108185bd07b9 100644 --- a/include/llvm/ADT/StringMap.h +++ b/include/llvm/ADT/StringMap.h @@ -118,36 +118,59 @@ public: } }; -/// StringMapEntry - This is used to represent one value that is inserted into -/// a StringMap. It contains the Value itself and the key: the string length -/// and data. +/// StringMapEntryStorage - Holds the value in a StringMapEntry. +/// +/// Factored out into a separate base class to make it easier to specialize. +/// This is primarily intended to support StringSet, which doesn't need a value +/// stored at all. template<typename ValueTy> -class StringMapEntry : public StringMapEntryBase { +class StringMapEntryStorage : public StringMapEntryBase { public: ValueTy second; - explicit StringMapEntry(size_t strLen) + explicit StringMapEntryStorage(size_t strLen) : StringMapEntryBase(strLen), second() {} template <typename... InitTy> - StringMapEntry(size_t strLen, InitTy &&... InitVals) + StringMapEntryStorage(size_t strLen, InitTy &&... InitVals) : StringMapEntryBase(strLen), second(std::forward<InitTy>(InitVals)...) {} - StringMapEntry(StringMapEntry &E) = delete; - - StringRef getKey() const { - return StringRef(getKeyData(), getKeyLength()); - } + StringMapEntryStorage(StringMapEntryStorage &E) = delete; const ValueTy &getValue() const { return second; } ValueTy &getValue() { return second; } void setValue(const ValueTy &V) { second = V; } +}; + +template<> +class StringMapEntryStorage<NoneType> : public StringMapEntryBase { +public: + explicit StringMapEntryStorage(size_t strLen, NoneType none = None) + : StringMapEntryBase(strLen) {} + StringMapEntryStorage(StringMapEntryStorage &E) = delete; + + NoneType getValue() const { return None; } +}; + +/// StringMapEntry - This is used to represent one value that is inserted into +/// a StringMap. It contains the Value itself and the key: the string length +/// and data. +template<typename ValueTy> +class StringMapEntry final : public StringMapEntryStorage<ValueTy> { +public: + using StringMapEntryStorage<ValueTy>::StringMapEntryStorage; + + StringRef getKey() const { + return StringRef(getKeyData(), this->getKeyLength()); + } /// getKeyData - Return the start of the string data that is the key for this /// value. The string data is always stored immediately after the /// StringMapEntry object. const char *getKeyData() const {return reinterpret_cast<const char*>(this+1);} - StringRef first() const { return StringRef(getKeyData(), getKeyLength()); } + StringRef first() const { + return StringRef(getKeyData(), this->getKeyLength()); + } /// Create a StringMapEntry for the specified key construct the value using /// \p InitiVals. @@ -199,7 +222,7 @@ public: template<typename AllocatorTy> void Destroy(AllocatorTy &Allocator) { // Free memory referenced by the item. - size_t AllocSize = sizeof(StringMapEntry) + getKeyLength() + 1; + size_t AllocSize = sizeof(StringMapEntry) + this->getKeyLength() + 1; this->~StringMapEntry(); Allocator.Deallocate(static_cast<void *>(this), AllocSize); } @@ -391,6 +414,16 @@ public: return try_emplace(KV.first, std::move(KV.second)); } + /// Inserts an element or assigns to the current element if the key already + /// exists. The return type is the same as try_emplace. + template <typename V> + std::pair<iterator, bool> insert_or_assign(StringRef Key, V &&Val) { + auto Ret = try_emplace(Key, std::forward<V>(Val)); + if (!Ret.second) + Ret.first->second = std::forward<V>(Val); + return Ret; + } + /// Emplace a new element for the specified key into the map if the key isn't /// already in the map. The bool component of the returned pair is true /// if and only if the insertion takes place, and the iterator component of diff --git a/include/llvm/ADT/StringRef.h b/include/llvm/ADT/StringRef.h index 4661b1e68b2f..52baab17bede 100644 --- a/include/llvm/ADT/StringRef.h +++ b/include/llvm/ADT/StringRef.h @@ -67,6 +67,20 @@ namespace llvm { return ::memcmp(Lhs,Rhs,Length); } + // Constexpr version of std::strlen. + static constexpr size_t strLen(const char *Str) { +#if __cplusplus > 201402L + return std::char_traits<char>::length(Str); +#elif __has_builtin(__builtin_strlen) || defined(__GNUC__) + return __builtin_strlen(Str); +#else + const char *Begin = Str; + while (*Str != '\0') + ++Str; + return Str - Begin; +#endif + } + public: /// @name Constructors /// @{ @@ -79,8 +93,8 @@ namespace llvm { StringRef(std::nullptr_t) = delete; /// Construct a string ref from a cstring. - /*implicit*/ StringRef(const char *Str) - : Data(Str), Length(Str ? ::strlen(Str) : 0) {} + /*implicit*/ constexpr StringRef(const char *Str) + : Data(Str), Length(Str ? strLen(Str) : 0) {} /// Construct a string ref from a pointer and length. /*implicit*/ constexpr StringRef(const char *data, size_t length) diff --git a/include/llvm/ADT/StringSet.h b/include/llvm/ADT/StringSet.h index af3a44a7b32c..60be09d3c326 100644 --- a/include/llvm/ADT/StringSet.h +++ b/include/llvm/ADT/StringSet.h @@ -24,8 +24,8 @@ namespace llvm { /// StringSet - A wrapper for StringMap that provides set-like functionality. template <class AllocatorTy = MallocAllocator> - class StringSet : public StringMap<char, AllocatorTy> { - using base = StringMap<char, AllocatorTy>; + class StringSet : public StringMap<NoneType, AllocatorTy> { + using base = StringMap<NoneType, AllocatorTy>; public: StringSet() = default; @@ -37,13 +37,13 @@ namespace llvm { std::pair<typename base::iterator, bool> insert(StringRef Key) { assert(!Key.empty()); - return base::insert(std::make_pair(Key, '\0')); + return base::insert(std::make_pair(Key, None)); } template <typename InputIt> void insert(const InputIt &Begin, const InputIt &End) { for (auto It = Begin; It != End; ++It) - base::insert(std::make_pair(*It, '\0')); + base::insert(std::make_pair(*It, None)); } template <typename ValueTy> diff --git a/include/llvm/ADT/TinyPtrVector.h b/include/llvm/ADT/TinyPtrVector.h index ac82451a9b21..6b76d35d4e92 100644 --- a/include/llvm/ADT/TinyPtrVector.h +++ b/include/llvm/ADT/TinyPtrVector.h @@ -31,6 +31,10 @@ class TinyPtrVector { public: using VecTy = SmallVector<EltTy, 4>; using value_type = typename VecTy::value_type; + // EltTy must be the first pointer type so that is<EltTy> is true for the + // default-constructed PtrUnion. This allows an empty TinyPtrVector to + // naturally vend a begin/end iterator of type EltTy* without an additional + // check for the empty state. using PtrUnion = PointerUnion<EltTy, VecTy *>; private: @@ -96,14 +100,14 @@ public: if (RHS.Val.template is<EltTy>()) { V->clear(); V->push_back(RHS.front()); - RHS.Val = (EltTy)nullptr; + RHS.Val = EltTy(); return *this; } delete V; } Val = RHS.Val; - RHS.Val = (EltTy)nullptr; + RHS.Val = EltTy(); return *this; } @@ -213,9 +217,9 @@ public: EltTy operator[](unsigned i) const { assert(!Val.isNull() && "can't index into an empty vector"); - if (EltTy V = Val.template dyn_cast<EltTy>()) { + if (Val.template is<EltTy>()) { assert(i == 0 && "tinyvector index out of range"); - return V; + return Val.template get<EltTy>(); } assert(i < Val.template get<VecTy*>()->size() && @@ -225,29 +229,29 @@ public: EltTy front() const { assert(!empty() && "vector empty"); - if (EltTy V = Val.template dyn_cast<EltTy>()) - return V; + if (Val.template is<EltTy>()) + return Val.template get<EltTy>(); return Val.template get<VecTy*>()->front(); } EltTy back() const { assert(!empty() && "vector empty"); - if (EltTy V = Val.template dyn_cast<EltTy>()) - return V; + if (Val.template is<EltTy>()) + return Val.template get<EltTy>(); return Val.template get<VecTy*>()->back(); } void push_back(EltTy NewVal) { - assert(NewVal && "Can't add a null value"); - // If we have nothing, add something. if (Val.isNull()) { Val = NewVal; + assert(!Val.isNull() && "Can't add a null value"); return; } // If we have a single value, convert to a vector. - if (EltTy V = Val.template dyn_cast<EltTy>()) { + if (Val.template is<EltTy>()) { + EltTy V = Val.template get<EltTy>(); Val = new VecTy(); Val.template get<VecTy*>()->push_back(V); } @@ -267,7 +271,7 @@ public: void clear() { // If we have a single value, convert to empty. if (Val.template is<EltTy>()) { - Val = (EltTy)nullptr; + Val = EltTy(); } else if (VecTy *Vec = Val.template dyn_cast<VecTy*>()) { // If we have a vector form, just clear it. Vec->clear(); @@ -282,7 +286,7 @@ public: // If we have a single value, convert to empty. if (Val.template is<EltTy>()) { if (I == begin()) - Val = (EltTy)nullptr; + Val = EltTy(); } else if (VecTy *Vec = Val.template dyn_cast<VecTy*>()) { // multiple items in a vector; just do the erase, there is no // benefit to collapsing back to a pointer @@ -298,7 +302,7 @@ public: if (Val.template is<EltTy>()) { if (S == begin() && S != E) - Val = (EltTy)nullptr; + Val = EltTy(); } else if (VecTy *Vec = Val.template dyn_cast<VecTy*>()) { return Vec->erase(S, E); } @@ -313,7 +317,8 @@ public: return std::prev(end()); } assert(!Val.isNull() && "Null value with non-end insert iterator."); - if (EltTy V = Val.template dyn_cast<EltTy>()) { + if (Val.template is<EltTy>()) { + EltTy V = Val.template get<EltTy>(); assert(I == begin()); Val = Elt; push_back(V); @@ -339,7 +344,8 @@ public: } Val = new VecTy(); - } else if (EltTy V = Val.template dyn_cast<EltTy>()) { + } else if (Val.template is<EltTy>()) { + EltTy V = Val.template get<EltTy>(); Val = new VecTy(); Val.template get<VecTy*>()->push_back(V); } diff --git a/include/llvm/ADT/VariadicFunction.h b/include/llvm/ADT/VariadicFunction.h deleted file mode 100644 index 5aefb05ecdda..000000000000 --- a/include/llvm/ADT/VariadicFunction.h +++ /dev/null @@ -1,330 +0,0 @@ -//===- VariadicFunction.h - Variadic Functions ------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file implements compile-time type-safe variadic functions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ADT_VARIADICFUNCTION_H -#define LLVM_ADT_VARIADICFUNCTION_H - -#include "llvm/ADT/ArrayRef.h" - -namespace llvm { - -// Define macros to aid in expanding a comma separated series with the index of -// the series pasted onto the last token. -#define LLVM_COMMA_JOIN1(x) x ## 0 -#define LLVM_COMMA_JOIN2(x) LLVM_COMMA_JOIN1(x), x ## 1 -#define LLVM_COMMA_JOIN3(x) LLVM_COMMA_JOIN2(x), x ## 2 -#define LLVM_COMMA_JOIN4(x) LLVM_COMMA_JOIN3(x), x ## 3 -#define LLVM_COMMA_JOIN5(x) LLVM_COMMA_JOIN4(x), x ## 4 -#define LLVM_COMMA_JOIN6(x) LLVM_COMMA_JOIN5(x), x ## 5 -#define LLVM_COMMA_JOIN7(x) LLVM_COMMA_JOIN6(x), x ## 6 -#define LLVM_COMMA_JOIN8(x) LLVM_COMMA_JOIN7(x), x ## 7 -#define LLVM_COMMA_JOIN9(x) LLVM_COMMA_JOIN8(x), x ## 8 -#define LLVM_COMMA_JOIN10(x) LLVM_COMMA_JOIN9(x), x ## 9 -#define LLVM_COMMA_JOIN11(x) LLVM_COMMA_JOIN10(x), x ## 10 -#define LLVM_COMMA_JOIN12(x) LLVM_COMMA_JOIN11(x), x ## 11 -#define LLVM_COMMA_JOIN13(x) LLVM_COMMA_JOIN12(x), x ## 12 -#define LLVM_COMMA_JOIN14(x) LLVM_COMMA_JOIN13(x), x ## 13 -#define LLVM_COMMA_JOIN15(x) LLVM_COMMA_JOIN14(x), x ## 14 -#define LLVM_COMMA_JOIN16(x) LLVM_COMMA_JOIN15(x), x ## 15 -#define LLVM_COMMA_JOIN17(x) LLVM_COMMA_JOIN16(x), x ## 16 -#define LLVM_COMMA_JOIN18(x) LLVM_COMMA_JOIN17(x), x ## 17 -#define LLVM_COMMA_JOIN19(x) LLVM_COMMA_JOIN18(x), x ## 18 -#define LLVM_COMMA_JOIN20(x) LLVM_COMMA_JOIN19(x), x ## 19 -#define LLVM_COMMA_JOIN21(x) LLVM_COMMA_JOIN20(x), x ## 20 -#define LLVM_COMMA_JOIN22(x) LLVM_COMMA_JOIN21(x), x ## 21 -#define LLVM_COMMA_JOIN23(x) LLVM_COMMA_JOIN22(x), x ## 22 -#define LLVM_COMMA_JOIN24(x) LLVM_COMMA_JOIN23(x), x ## 23 -#define LLVM_COMMA_JOIN25(x) LLVM_COMMA_JOIN24(x), x ## 24 -#define LLVM_COMMA_JOIN26(x) LLVM_COMMA_JOIN25(x), x ## 25 -#define LLVM_COMMA_JOIN27(x) LLVM_COMMA_JOIN26(x), x ## 26 -#define LLVM_COMMA_JOIN28(x) LLVM_COMMA_JOIN27(x), x ## 27 -#define LLVM_COMMA_JOIN29(x) LLVM_COMMA_JOIN28(x), x ## 28 -#define LLVM_COMMA_JOIN30(x) LLVM_COMMA_JOIN29(x), x ## 29 -#define LLVM_COMMA_JOIN31(x) LLVM_COMMA_JOIN30(x), x ## 30 -#define LLVM_COMMA_JOIN32(x) LLVM_COMMA_JOIN31(x), x ## 31 - -/// Class which can simulate a type-safe variadic function. -/// -/// The VariadicFunction class template makes it easy to define -/// type-safe variadic functions where all arguments have the same -/// type. -/// -/// Suppose we need a variadic function like this: -/// -/// ResultT Foo(const ArgT &A_0, const ArgT &A_1, ..., const ArgT &A_N); -/// -/// Instead of many overloads of Foo(), we only need to define a helper -/// function that takes an array of arguments: -/// -/// ResultT FooImpl(ArrayRef<const ArgT *> Args) { -/// // 'Args[i]' is a pointer to the i-th argument passed to Foo(). -/// ... -/// } -/// -/// and then define Foo() like this: -/// -/// const VariadicFunction<ResultT, ArgT, FooImpl> Foo; -/// -/// VariadicFunction takes care of defining the overloads of Foo(). -/// -/// Actually, Foo is a function object (i.e. functor) instead of a plain -/// function. This object is stateless and its constructor/destructor -/// does nothing, so it's safe to create global objects and call Foo(...) at -/// any time. -/// -/// Sometimes we need a variadic function to have some fixed leading -/// arguments whose types may be different from that of the optional -/// arguments. For example: -/// -/// bool FullMatch(const StringRef &S, const RE &Regex, -/// const ArgT &A_0, ..., const ArgT &A_N); -/// -/// VariadicFunctionN is for such cases, where N is the number of fixed -/// arguments. It is like VariadicFunction, except that it takes N more -/// template arguments for the types of the fixed arguments: -/// -/// bool FullMatchImpl(const StringRef &S, const RE &Regex, -/// ArrayRef<const ArgT *> Args) { ... } -/// const VariadicFunction2<bool, const StringRef&, -/// const RE&, ArgT, FullMatchImpl> -/// FullMatch; -/// -/// Currently VariadicFunction and friends support up-to 3 -/// fixed leading arguments and up-to 32 optional arguments. -template <typename ResultT, typename ArgT, - ResultT (*Func)(ArrayRef<const ArgT *>)> -struct VariadicFunction { - ResultT operator()() const { - return Func(None); - } - -#define LLVM_DEFINE_OVERLOAD(N) \ - ResultT operator()(LLVM_COMMA_JOIN ## N(const ArgT &A)) const { \ - const ArgT *const Args[] = { LLVM_COMMA_JOIN ## N(&A) }; \ - return Func(makeArrayRef(Args)); \ - } - LLVM_DEFINE_OVERLOAD(1) - LLVM_DEFINE_OVERLOAD(2) - LLVM_DEFINE_OVERLOAD(3) - LLVM_DEFINE_OVERLOAD(4) - LLVM_DEFINE_OVERLOAD(5) - LLVM_DEFINE_OVERLOAD(6) - LLVM_DEFINE_OVERLOAD(7) - LLVM_DEFINE_OVERLOAD(8) - LLVM_DEFINE_OVERLOAD(9) - LLVM_DEFINE_OVERLOAD(10) - LLVM_DEFINE_OVERLOAD(11) - LLVM_DEFINE_OVERLOAD(12) - LLVM_DEFINE_OVERLOAD(13) - LLVM_DEFINE_OVERLOAD(14) - LLVM_DEFINE_OVERLOAD(15) - LLVM_DEFINE_OVERLOAD(16) - LLVM_DEFINE_OVERLOAD(17) - LLVM_DEFINE_OVERLOAD(18) - LLVM_DEFINE_OVERLOAD(19) - LLVM_DEFINE_OVERLOAD(20) - LLVM_DEFINE_OVERLOAD(21) - LLVM_DEFINE_OVERLOAD(22) - LLVM_DEFINE_OVERLOAD(23) - LLVM_DEFINE_OVERLOAD(24) - LLVM_DEFINE_OVERLOAD(25) - LLVM_DEFINE_OVERLOAD(26) - LLVM_DEFINE_OVERLOAD(27) - LLVM_DEFINE_OVERLOAD(28) - LLVM_DEFINE_OVERLOAD(29) - LLVM_DEFINE_OVERLOAD(30) - LLVM_DEFINE_OVERLOAD(31) - LLVM_DEFINE_OVERLOAD(32) -#undef LLVM_DEFINE_OVERLOAD -}; - -template <typename ResultT, typename Param0T, typename ArgT, - ResultT (*Func)(Param0T, ArrayRef<const ArgT *>)> -struct VariadicFunction1 { - ResultT operator()(Param0T P0) const { - return Func(P0, None); - } - -#define LLVM_DEFINE_OVERLOAD(N) \ - ResultT operator()(Param0T P0, LLVM_COMMA_JOIN ## N(const ArgT &A)) const { \ - const ArgT *const Args[] = { LLVM_COMMA_JOIN ## N(&A) }; \ - return Func(P0, makeArrayRef(Args)); \ - } - LLVM_DEFINE_OVERLOAD(1) - LLVM_DEFINE_OVERLOAD(2) - LLVM_DEFINE_OVERLOAD(3) - LLVM_DEFINE_OVERLOAD(4) - LLVM_DEFINE_OVERLOAD(5) - LLVM_DEFINE_OVERLOAD(6) - LLVM_DEFINE_OVERLOAD(7) - LLVM_DEFINE_OVERLOAD(8) - LLVM_DEFINE_OVERLOAD(9) - LLVM_DEFINE_OVERLOAD(10) - LLVM_DEFINE_OVERLOAD(11) - LLVM_DEFINE_OVERLOAD(12) - LLVM_DEFINE_OVERLOAD(13) - LLVM_DEFINE_OVERLOAD(14) - LLVM_DEFINE_OVERLOAD(15) - LLVM_DEFINE_OVERLOAD(16) - LLVM_DEFINE_OVERLOAD(17) - LLVM_DEFINE_OVERLOAD(18) - LLVM_DEFINE_OVERLOAD(19) - LLVM_DEFINE_OVERLOAD(20) - LLVM_DEFINE_OVERLOAD(21) - LLVM_DEFINE_OVERLOAD(22) - LLVM_DEFINE_OVERLOAD(23) - LLVM_DEFINE_OVERLOAD(24) - LLVM_DEFINE_OVERLOAD(25) - LLVM_DEFINE_OVERLOAD(26) - LLVM_DEFINE_OVERLOAD(27) - LLVM_DEFINE_OVERLOAD(28) - LLVM_DEFINE_OVERLOAD(29) - LLVM_DEFINE_OVERLOAD(30) - LLVM_DEFINE_OVERLOAD(31) - LLVM_DEFINE_OVERLOAD(32) -#undef LLVM_DEFINE_OVERLOAD -}; - -template <typename ResultT, typename Param0T, typename Param1T, typename ArgT, - ResultT (*Func)(Param0T, Param1T, ArrayRef<const ArgT *>)> -struct VariadicFunction2 { - ResultT operator()(Param0T P0, Param1T P1) const { - return Func(P0, P1, None); - } - -#define LLVM_DEFINE_OVERLOAD(N) \ - ResultT operator()(Param0T P0, Param1T P1, \ - LLVM_COMMA_JOIN ## N(const ArgT &A)) const { \ - const ArgT *const Args[] = { LLVM_COMMA_JOIN ## N(&A) }; \ - return Func(P0, P1, makeArrayRef(Args)); \ - } - LLVM_DEFINE_OVERLOAD(1) - LLVM_DEFINE_OVERLOAD(2) - LLVM_DEFINE_OVERLOAD(3) - LLVM_DEFINE_OVERLOAD(4) - LLVM_DEFINE_OVERLOAD(5) - LLVM_DEFINE_OVERLOAD(6) - LLVM_DEFINE_OVERLOAD(7) - LLVM_DEFINE_OVERLOAD(8) - LLVM_DEFINE_OVERLOAD(9) - LLVM_DEFINE_OVERLOAD(10) - LLVM_DEFINE_OVERLOAD(11) - LLVM_DEFINE_OVERLOAD(12) - LLVM_DEFINE_OVERLOAD(13) - LLVM_DEFINE_OVERLOAD(14) - LLVM_DEFINE_OVERLOAD(15) - LLVM_DEFINE_OVERLOAD(16) - LLVM_DEFINE_OVERLOAD(17) - LLVM_DEFINE_OVERLOAD(18) - LLVM_DEFINE_OVERLOAD(19) - LLVM_DEFINE_OVERLOAD(20) - LLVM_DEFINE_OVERLOAD(21) - LLVM_DEFINE_OVERLOAD(22) - LLVM_DEFINE_OVERLOAD(23) - LLVM_DEFINE_OVERLOAD(24) - LLVM_DEFINE_OVERLOAD(25) - LLVM_DEFINE_OVERLOAD(26) - LLVM_DEFINE_OVERLOAD(27) - LLVM_DEFINE_OVERLOAD(28) - LLVM_DEFINE_OVERLOAD(29) - LLVM_DEFINE_OVERLOAD(30) - LLVM_DEFINE_OVERLOAD(31) - LLVM_DEFINE_OVERLOAD(32) -#undef LLVM_DEFINE_OVERLOAD -}; - -template <typename ResultT, typename Param0T, typename Param1T, - typename Param2T, typename ArgT, - ResultT (*Func)(Param0T, Param1T, Param2T, ArrayRef<const ArgT *>)> -struct VariadicFunction3 { - ResultT operator()(Param0T P0, Param1T P1, Param2T P2) const { - return Func(P0, P1, P2, None); - } - -#define LLVM_DEFINE_OVERLOAD(N) \ - ResultT operator()(Param0T P0, Param1T P1, Param2T P2, \ - LLVM_COMMA_JOIN ## N(const ArgT &A)) const { \ - const ArgT *const Args[] = { LLVM_COMMA_JOIN ## N(&A) }; \ - return Func(P0, P1, P2, makeArrayRef(Args)); \ - } - LLVM_DEFINE_OVERLOAD(1) - LLVM_DEFINE_OVERLOAD(2) - LLVM_DEFINE_OVERLOAD(3) - LLVM_DEFINE_OVERLOAD(4) - LLVM_DEFINE_OVERLOAD(5) - LLVM_DEFINE_OVERLOAD(6) - LLVM_DEFINE_OVERLOAD(7) - LLVM_DEFINE_OVERLOAD(8) - LLVM_DEFINE_OVERLOAD(9) - LLVM_DEFINE_OVERLOAD(10) - LLVM_DEFINE_OVERLOAD(11) - LLVM_DEFINE_OVERLOAD(12) - LLVM_DEFINE_OVERLOAD(13) - LLVM_DEFINE_OVERLOAD(14) - LLVM_DEFINE_OVERLOAD(15) - LLVM_DEFINE_OVERLOAD(16) - LLVM_DEFINE_OVERLOAD(17) - LLVM_DEFINE_OVERLOAD(18) - LLVM_DEFINE_OVERLOAD(19) - LLVM_DEFINE_OVERLOAD(20) - LLVM_DEFINE_OVERLOAD(21) - LLVM_DEFINE_OVERLOAD(22) - LLVM_DEFINE_OVERLOAD(23) - LLVM_DEFINE_OVERLOAD(24) - LLVM_DEFINE_OVERLOAD(25) - LLVM_DEFINE_OVERLOAD(26) - LLVM_DEFINE_OVERLOAD(27) - LLVM_DEFINE_OVERLOAD(28) - LLVM_DEFINE_OVERLOAD(29) - LLVM_DEFINE_OVERLOAD(30) - LLVM_DEFINE_OVERLOAD(31) - LLVM_DEFINE_OVERLOAD(32) -#undef LLVM_DEFINE_OVERLOAD -}; - -// Cleanup the macro namespace. -#undef LLVM_COMMA_JOIN1 -#undef LLVM_COMMA_JOIN2 -#undef LLVM_COMMA_JOIN3 -#undef LLVM_COMMA_JOIN4 -#undef LLVM_COMMA_JOIN5 -#undef LLVM_COMMA_JOIN6 -#undef LLVM_COMMA_JOIN7 -#undef LLVM_COMMA_JOIN8 -#undef LLVM_COMMA_JOIN9 -#undef LLVM_COMMA_JOIN10 -#undef LLVM_COMMA_JOIN11 -#undef LLVM_COMMA_JOIN12 -#undef LLVM_COMMA_JOIN13 -#undef LLVM_COMMA_JOIN14 -#undef LLVM_COMMA_JOIN15 -#undef LLVM_COMMA_JOIN16 -#undef LLVM_COMMA_JOIN17 -#undef LLVM_COMMA_JOIN18 -#undef LLVM_COMMA_JOIN19 -#undef LLVM_COMMA_JOIN20 -#undef LLVM_COMMA_JOIN21 -#undef LLVM_COMMA_JOIN22 -#undef LLVM_COMMA_JOIN23 -#undef LLVM_COMMA_JOIN24 -#undef LLVM_COMMA_JOIN25 -#undef LLVM_COMMA_JOIN26 -#undef LLVM_COMMA_JOIN27 -#undef LLVM_COMMA_JOIN28 -#undef LLVM_COMMA_JOIN29 -#undef LLVM_COMMA_JOIN30 -#undef LLVM_COMMA_JOIN31 -#undef LLVM_COMMA_JOIN32 - -} // end namespace llvm - -#endif // LLVM_ADT_VARIADICFUNCTION_H diff --git a/include/llvm/ADT/iterator_range.h b/include/llvm/ADT/iterator_range.h index 774c7c4e3366..aa8830943cab 100644 --- a/include/llvm/ADT/iterator_range.h +++ b/include/llvm/ADT/iterator_range.h @@ -44,6 +44,7 @@ public: IteratorT begin() const { return begin_iterator; } IteratorT end() const { return end_iterator; } + bool empty() const { return begin_iterator == end_iterator; } }; /// Convenience function for iterating over sub-ranges. diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h index 948341554f23..282142f51bb3 100644 --- a/include/llvm/Analysis/AliasAnalysis.h +++ b/include/llvm/Analysis/AliasAnalysis.h @@ -949,7 +949,7 @@ template <typename DerivedT> class AAResultBase { /// A pointer to the AAResults object that this AAResult is /// aggregated within. May be null if not aggregated. - AAResults *AAR; + AAResults *AAR = nullptr; /// Helper to dispatch calls back through the derived type. DerivedT &derived() { return static_cast<DerivedT &>(*this); } diff --git a/include/llvm/Analysis/AliasSetTracker.h b/include/llvm/Analysis/AliasSetTracker.h index 34a509b7f4bb..187317e3831b 100644 --- a/include/llvm/Analysis/AliasSetTracker.h +++ b/include/llvm/Analysis/AliasSetTracker.h @@ -87,10 +87,11 @@ class AliasSet : public ilist_node<AliasSet> { AAInfo = NewAAInfo; else { AAMDNodes Intersection(AAInfo.intersect(NewAAInfo)); - if (!Intersection) { + if (!Intersection.TBAA || !Intersection.Scope || + !Intersection.NoAlias) { // NewAAInfo conflicts with AAInfo. AAInfo = DenseMapInfo<AAMDNodes>::getTombstoneKey(); - return SizeChanged; + SizeChanged = true; } AAInfo = Intersection; } diff --git a/include/llvm/Analysis/AssumptionCache.h b/include/llvm/Analysis/AssumptionCache.h index b42846472f2e..0efbd59023d6 100644 --- a/include/llvm/Analysis/AssumptionCache.h +++ b/include/llvm/Analysis/AssumptionCache.h @@ -73,8 +73,8 @@ class AssumptionCache { /// Get the vector of assumptions which affect a value from the cache. SmallVector<WeakTrackingVH, 1> &getOrInsertAffectedValues(Value *V); - /// Copy affected values in the cache for OV to be affected values for NV. - void copyAffectedValuesInCache(Value *OV, Value *NV); + /// Move affected values in the cache for OV to be affected values for NV. + void transferAffectedValuesInCache(Value *OV, Value *NV); /// Flag tracking whether we have scanned the function yet. /// diff --git a/include/llvm/Analysis/CFG.h b/include/llvm/Analysis/CFG.h index bb55e76ac86a..68f137ba622c 100644 --- a/include/llvm/Analysis/CFG.h +++ b/include/llvm/Analysis/CFG.h @@ -46,6 +46,8 @@ unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ); /// bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges = false); +bool isCriticalEdge(const Instruction *TI, const BasicBlock *Succ, + bool AllowIdenticalEdges = false); /// Determine whether instruction 'To' is reachable from 'From', without passing /// through any blocks in ExclusionSet, returning true if uncertain. diff --git a/include/llvm/Analysis/CFLAndersAliasAnalysis.h b/include/llvm/Analysis/CFLAndersAliasAnalysis.h index 7c8b42b1d8d2..5f5e52af3d88 100644 --- a/include/llvm/Analysis/CFLAndersAliasAnalysis.h +++ b/include/llvm/Analysis/CFLAndersAliasAnalysis.h @@ -41,7 +41,8 @@ class CFLAndersAAResult : public AAResultBase<CFLAndersAAResult> { class FunctionInfo; public: - explicit CFLAndersAAResult(const TargetLibraryInfo &TLI); + explicit CFLAndersAAResult( + std::function<const TargetLibraryInfo &(Function &F)> GetTLI); CFLAndersAAResult(CFLAndersAAResult &&RHS); ~CFLAndersAAResult(); @@ -74,7 +75,7 @@ private: /// Build summary for a given function FunctionInfo buildInfoFrom(const Function &); - const TargetLibraryInfo &TLI; + std::function<const TargetLibraryInfo &(Function &F)> GetTLI; /// Cached mapping of Functions to their StratifiedSets. /// If a function's sets are currently being built, it is marked diff --git a/include/llvm/Analysis/CFLSteensAliasAnalysis.h b/include/llvm/Analysis/CFLSteensAliasAnalysis.h index cc7a47cd9a5f..135321616b7c 100644 --- a/include/llvm/Analysis/CFLSteensAliasAnalysis.h +++ b/include/llvm/Analysis/CFLSteensAliasAnalysis.h @@ -42,7 +42,8 @@ class CFLSteensAAResult : public AAResultBase<CFLSteensAAResult> { class FunctionInfo; public: - explicit CFLSteensAAResult(const TargetLibraryInfo &TLI); + explicit CFLSteensAAResult( + std::function<const TargetLibraryInfo &(Function &)> GetTLI); CFLSteensAAResult(CFLSteensAAResult &&Arg); ~CFLSteensAAResult(); @@ -90,7 +91,7 @@ public: } private: - const TargetLibraryInfo &TLI; + std::function<const TargetLibraryInfo &(Function &)> GetTLI; /// Cached mapping of Functions to their StratifiedSets. /// If a function's sets are currently being built, it is marked diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h index 8af5fb86995a..933f2210dafc 100644 --- a/include/llvm/Analysis/CGSCCPassManager.h +++ b/include/llvm/Analysis/CGSCCPassManager.h @@ -88,6 +88,7 @@ #ifndef LLVM_ANALYSIS_CGSCCPASSMANAGER_H #define LLVM_ANALYSIS_CGSCCPASSMANAGER_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/PriorityWorklist.h" #include "llvm/ADT/STLExtras.h" @@ -583,10 +584,12 @@ public: SmallVectorImpl<WeakTrackingVH> &CallHandles) { assert(CallHandles.empty() && "Must start with a clear set of handles."); - SmallVector<CallCount, 4> CallCounts; + SmallDenseMap<Function *, CallCount> CallCounts; + CallCount CountLocal = {0, 0}; for (LazyCallGraph::Node &N : C) { - CallCounts.push_back({0, 0}); - CallCount &Count = CallCounts.back(); + CallCount &Count = + CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal)) + .first->second; for (Instruction &I : instructions(N.getFunction())) if (auto CS = CallSite(&I)) { if (CS.getCalledFunction()) { @@ -626,8 +629,6 @@ public: // Check that we didn't miss any update scenario. assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!"); assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - assert((int)CallCounts.size() == C->size() && - "Cannot have changed the size of the SCC!"); // Check whether any of the handles were devirtualized. auto IsDevirtualizedHandle = [&](WeakTrackingVH &CallH) { @@ -642,7 +643,7 @@ public: if (!F) return false; - LLVM_DEBUG(dbgs() << "Found devirutalized call from " + LLVM_DEBUG(dbgs() << "Found devirtualized call from " << CS.getParent()->getParent()->getName() << " to " << F->getName() << "\n"); @@ -664,12 +665,20 @@ public: // manner of transformations such as DCE and other things, but seems to // work well in practice. if (!Devirt) - for (int i = 0, Size = C->size(); i < Size; ++i) - if (CallCounts[i].Indirect > NewCallCounts[i].Indirect && - CallCounts[i].Direct < NewCallCounts[i].Direct) { - Devirt = true; - break; + // Iterate over the keys in NewCallCounts, if Function also exists in + // CallCounts, make the check below. + for (auto &Pair : NewCallCounts) { + auto &CallCountNew = Pair.second; + auto CountIt = CallCounts.find(Pair.first); + if (CountIt != CallCounts.end()) { + const auto &CallCountOld = CountIt->second; + if (CallCountOld.Indirect > CallCountNew.Indirect && + CallCountOld.Direct < CallCountNew.Direct) { + Devirt = true; + break; + } } + } if (!Devirt) { PA.intersect(std::move(PassPA)); diff --git a/include/llvm/Analysis/CaptureTracking.h b/include/llvm/Analysis/CaptureTracking.h index ca7abd34fea2..29921a51d5be 100644 --- a/include/llvm/Analysis/CaptureTracking.h +++ b/include/llvm/Analysis/CaptureTracking.h @@ -17,6 +17,7 @@ namespace llvm { class Value; class Use; + class DataLayout; class Instruction; class DominatorTree; class OrderedBasicBlock; @@ -83,6 +84,11 @@ namespace llvm { /// use U. Return true to stop the traversal or false to continue looking /// for more capturing instructions. virtual bool captured(const Use *U) = 0; + + /// isDereferenceableOrNull - Overload to allow clients with additional + /// knowledge about pointer dereferenceability to provide it and thereby + /// avoid conservative responses when a pointer is compared to null. + virtual bool isDereferenceableOrNull(Value *O, const DataLayout &DL); }; /// PointerMayBeCaptured - Visit the value and the values derived from it and diff --git a/include/llvm/Analysis/DDG.h b/include/llvm/Analysis/DDG.h new file mode 100644 index 000000000000..0e1eb9d2cda3 --- /dev/null +++ b/include/llvm/Analysis/DDG.h @@ -0,0 +1,430 @@ +//===- llvm/Analysis/DDG.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the Data-Dependence Graph (DDG). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DDG_H +#define LLVM_ANALYSIS_DDG_H + +#include "llvm/ADT/DirectedGraph.h" +#include "llvm/Analysis/DependenceAnalysis.h" +#include "llvm/Analysis/DependenceGraphBuilder.h" +#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/IR/Instructions.h" +#include <unordered_map> + +namespace llvm { +class DDGNode; +class DDGEdge; +using DDGNodeBase = DGNode<DDGNode, DDGEdge>; +using DDGEdgeBase = DGEdge<DDGNode, DDGEdge>; +using DDGBase = DirectedGraph<DDGNode, DDGEdge>; +class LPMUpdater; + +/// Data Dependence Graph Node +/// The graph can represent the following types of nodes: +/// 1. Single instruction node containing just one instruction. +/// 2. Multiple instruction node where two or more instructions from +/// the same basic block are merged into one node. +/// 3. Root node is a special node that connects to all components such that +/// there is always a path from it to any node in the graph. +class DDGNode : public DDGNodeBase { +public: + using InstructionListType = SmallVectorImpl<Instruction *>; + + enum class NodeKind { + Unknown, + SingleInstruction, + MultiInstruction, + Root, + }; + + DDGNode() = delete; + DDGNode(const NodeKind K) : DDGNodeBase(), Kind(K) {} + DDGNode(const DDGNode &N) : DDGNodeBase(N), Kind(N.Kind) {} + DDGNode(DDGNode &&N) : DDGNodeBase(std::move(N)), Kind(N.Kind) {} + virtual ~DDGNode() = 0; + + DDGNode &operator=(const DDGNode &N) { + DGNode::operator=(N); + Kind = N.Kind; + return *this; + } + + DDGNode &operator=(DDGNode &&N) { + DGNode::operator=(std::move(N)); + Kind = N.Kind; + return *this; + } + + /// Getter for the kind of this node. + NodeKind getKind() const { return Kind; } + + /// Collect a list of instructions, in \p IList, for which predicate \p Pred + /// evaluates to true when iterating over instructions of this node. Return + /// true if at least one instruction was collected, and false otherwise. + bool collectInstructions(llvm::function_ref<bool(Instruction *)> const &Pred, + InstructionListType &IList) const; + +protected: + /// Setter for the kind of this node. + void setKind(NodeKind K) { Kind = K; } + +private: + NodeKind Kind; +}; + +/// Subclass of DDGNode representing the root node of the graph. +/// There should only be one such node in a given graph. +class RootDDGNode : public DDGNode { +public: + RootDDGNode() : DDGNode(NodeKind::Root) {} + RootDDGNode(const RootDDGNode &N) = delete; + RootDDGNode(RootDDGNode &&N) : DDGNode(std::move(N)) {} + ~RootDDGNode() {} + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const DDGNode *N) { + return N->getKind() == NodeKind::Root; + } + static bool classof(const RootDDGNode *N) { return true; } +}; + +/// Subclass of DDGNode representing single or multi-instruction nodes. +class SimpleDDGNode : public DDGNode { +public: + SimpleDDGNode() = delete; + SimpleDDGNode(Instruction &I); + SimpleDDGNode(const SimpleDDGNode &N); + SimpleDDGNode(SimpleDDGNode &&N); + ~SimpleDDGNode(); + + SimpleDDGNode &operator=(const SimpleDDGNode &N) { + DDGNode::operator=(N); + InstList = N.InstList; + return *this; + } + + SimpleDDGNode &operator=(SimpleDDGNode &&N) { + DDGNode::operator=(std::move(N)); + InstList = std::move(N.InstList); + return *this; + } + + /// Get the list of instructions in this node. + const InstructionListType &getInstructions() const { + assert(!InstList.empty() && "Instruction List is empty."); + return InstList; + } + InstructionListType &getInstructions() { + return const_cast<InstructionListType &>( + static_cast<const SimpleDDGNode *>(this)->getInstructions()); + } + + /// Get the first/last instruction in the node. + Instruction *getFirstInstruction() const { return getInstructions().front(); } + Instruction *getLastInstruction() const { return getInstructions().back(); } + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const DDGNode *N) { + return N->getKind() == NodeKind::SingleInstruction || + N->getKind() == NodeKind::MultiInstruction; + } + static bool classof(const SimpleDDGNode *N) { return true; } + +private: + /// Append the list of instructions in \p Input to this node. + void appendInstructions(const InstructionListType &Input) { + setKind((InstList.size() == 0 && Input.size() == 1) + ? NodeKind::SingleInstruction + : NodeKind::MultiInstruction); + InstList.insert(InstList.end(), Input.begin(), Input.end()); + } + void appendInstructions(const SimpleDDGNode &Input) { + appendInstructions(Input.getInstructions()); + } + + /// List of instructions associated with a single or multi-instruction node. + SmallVector<Instruction *, 2> InstList; +}; + +/// Data Dependency Graph Edge. +/// An edge in the DDG can represent a def-use relationship or +/// a memory dependence based on the result of DependenceAnalysis. +/// A rooted edge connects the root node to one of the components +/// of the graph. +class DDGEdge : public DDGEdgeBase { +public: + /// The kind of edge in the DDG + enum class EdgeKind { Unknown, RegisterDefUse, MemoryDependence, Rooted }; + + explicit DDGEdge(DDGNode &N) = delete; + DDGEdge(DDGNode &N, EdgeKind K) : DDGEdgeBase(N), Kind(K) {} + DDGEdge(const DDGEdge &E) : DDGEdgeBase(E), Kind(E.getKind()) {} + DDGEdge(DDGEdge &&E) : DDGEdgeBase(std::move(E)), Kind(E.Kind) {} + DDGEdge &operator=(const DDGEdge &E) { + DDGEdgeBase::operator=(E); + Kind = E.Kind; + return *this; + } + + DDGEdge &operator=(DDGEdge &&E) { + DDGEdgeBase::operator=(std::move(E)); + Kind = E.Kind; + return *this; + } + + /// Get the edge kind + EdgeKind getKind() const { return Kind; }; + + /// Return true if this is a def-use edge, and false otherwise. + bool isDefUse() const { return Kind == EdgeKind::RegisterDefUse; } + + /// Return true if this is a memory dependence edge, and false otherwise. + bool isMemoryDependence() const { return Kind == EdgeKind::MemoryDependence; } + + /// Return true if this is an edge stemming from the root node, and false + /// otherwise. + bool isRooted() const { return Kind == EdgeKind::Rooted; } + +private: + EdgeKind Kind; +}; + +/// Encapsulate some common data and functionality needed for different +/// variations of data dependence graphs. +template <typename NodeType> class DependenceGraphInfo { +public: + using DependenceList = SmallVector<std::unique_ptr<Dependence>, 1>; + + DependenceGraphInfo() = delete; + DependenceGraphInfo(const DependenceGraphInfo &G) = delete; + DependenceGraphInfo(const std::string &N, const DependenceInfo &DepInfo) + : Name(N), DI(DepInfo), Root(nullptr) {} + DependenceGraphInfo(DependenceGraphInfo &&G) + : Name(std::move(G.Name)), DI(std::move(G.DI)), Root(G.Root) {} + virtual ~DependenceGraphInfo() {} + + /// Return the label that is used to name this graph. + const StringRef getName() const { return Name; } + + /// Return the root node of the graph. + NodeType &getRoot() const { + assert(Root && "Root node is not available yet. Graph construction may " + "still be in progress\n"); + return *Root; + } + +protected: + // Name of the graph. + std::string Name; + + // Store a copy of DependenceInfo in the graph, so that individual memory + // dependencies don't need to be stored. Instead when the dependence is + // queried it is recomputed using @DI. + const DependenceInfo DI; + + // A special node in the graph that has an edge to every connected component of + // the graph, to ensure all nodes are reachable in a graph walk. + NodeType *Root = nullptr; +}; + +using DDGInfo = DependenceGraphInfo<DDGNode>; + +/// Data Dependency Graph +class DataDependenceGraph : public DDGBase, public DDGInfo { + friend class DDGBuilder; + +public: + using NodeType = DDGNode; + using EdgeType = DDGEdge; + + DataDependenceGraph() = delete; + DataDependenceGraph(const DataDependenceGraph &G) = delete; + DataDependenceGraph(DataDependenceGraph &&G) + : DDGBase(std::move(G)), DDGInfo(std::move(G)) {} + DataDependenceGraph(Function &F, DependenceInfo &DI); + DataDependenceGraph(const Loop &L, DependenceInfo &DI); + ~DataDependenceGraph(); + +protected: + /// Add node \p N to the graph, if it's not added yet, and keep track of + /// the root node. Return true if node is successfully added. + bool addNode(NodeType &N); + +}; + +/// Concrete implementation of a pure data dependence graph builder. This class +/// provides custom implementation for the pure-virtual functions used in the +/// generic dependence graph build algorithm. +/// +/// For information about time complexity of the build algorithm see the +/// comments near the declaration of AbstractDependenceGraphBuilder. +class DDGBuilder : public AbstractDependenceGraphBuilder<DataDependenceGraph> { +public: + DDGBuilder(DataDependenceGraph &G, DependenceInfo &D, + const BasicBlockListType &BBs) + : AbstractDependenceGraphBuilder(G, D, BBs) {} + DDGNode &createRootNode() final override { + auto *RN = new RootDDGNode(); + assert(RN && "Failed to allocate memory for DDG root node."); + Graph.addNode(*RN); + return *RN; + } + DDGNode &createFineGrainedNode(Instruction &I) final override { + auto *SN = new SimpleDDGNode(I); + assert(SN && "Failed to allocate memory for simple DDG node."); + Graph.addNode(*SN); + return *SN; + } + DDGEdge &createDefUseEdge(DDGNode &Src, DDGNode &Tgt) final override { + auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::RegisterDefUse); + assert(E && "Failed to allocate memory for edge"); + Graph.connect(Src, Tgt, *E); + return *E; + } + DDGEdge &createMemoryEdge(DDGNode &Src, DDGNode &Tgt) final override { + auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::MemoryDependence); + assert(E && "Failed to allocate memory for edge"); + Graph.connect(Src, Tgt, *E); + return *E; + } + DDGEdge &createRootedEdge(DDGNode &Src, DDGNode &Tgt) final override { + auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::Rooted); + assert(E && "Failed to allocate memory for edge"); + assert(isa<RootDDGNode>(Src) && "Expected root node"); + Graph.connect(Src, Tgt, *E); + return *E; + } + +}; + +raw_ostream &operator<<(raw_ostream &OS, const DDGNode &N); +raw_ostream &operator<<(raw_ostream &OS, const DDGNode::NodeKind K); +raw_ostream &operator<<(raw_ostream &OS, const DDGEdge &E); +raw_ostream &operator<<(raw_ostream &OS, const DDGEdge::EdgeKind K); +raw_ostream &operator<<(raw_ostream &OS, const DataDependenceGraph &G); + +//===--------------------------------------------------------------------===// +// DDG Analysis Passes +//===--------------------------------------------------------------------===// + +/// Analysis pass that builds the DDG for a loop. +class DDGAnalysis : public AnalysisInfoMixin<DDGAnalysis> { +public: + using Result = std::unique_ptr<DataDependenceGraph>; + Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR); + +private: + friend AnalysisInfoMixin<DDGAnalysis>; + static AnalysisKey Key; +}; + +/// Textual printer pass for the DDG of a loop. +class DDGAnalysisPrinterPass : public PassInfoMixin<DDGAnalysisPrinterPass> { +public: + explicit DDGAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); + +private: + raw_ostream &OS; +}; + +//===--------------------------------------------------------------------===// +// GraphTraits specializations for the DDG +//===--------------------------------------------------------------------===// + +/// non-const versions of the grapth trait specializations for DDG +template <> struct GraphTraits<DDGNode *> { + using NodeRef = DDGNode *; + + static DDGNode *DDGGetTargetNode(DGEdge<DDGNode, DDGEdge> *P) { + return &P->getTargetNode(); + } + + // Provide a mapped iterator so that the GraphTrait-based implementations can + // find the target nodes without having to explicitly go through the edges. + using ChildIteratorType = + mapped_iterator<DDGNode::iterator, decltype(&DDGGetTargetNode)>; + using ChildEdgeIteratorType = DDGNode::iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &DDGGetTargetNode); + } + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &DDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits<DataDependenceGraph *> : public GraphTraits<DDGNode *> { + using nodes_iterator = DataDependenceGraph::iterator; + static NodeRef getEntryNode(DataDependenceGraph *DG) { + return &DG->getRoot(); + } + static nodes_iterator nodes_begin(DataDependenceGraph *DG) { + return DG->begin(); + } + static nodes_iterator nodes_end(DataDependenceGraph *DG) { return DG->end(); } +}; + +/// const versions of the grapth trait specializations for DDG +template <> struct GraphTraits<const DDGNode *> { + using NodeRef = const DDGNode *; + + static const DDGNode *DDGGetTargetNode(const DGEdge<DDGNode, DDGEdge> *P) { + return &P->getTargetNode(); + } + + // Provide a mapped iterator so that the GraphTrait-based implementations can + // find the target nodes without having to explicitly go through the edges. + using ChildIteratorType = + mapped_iterator<DDGNode::const_iterator, decltype(&DDGGetTargetNode)>; + using ChildEdgeIteratorType = DDGNode::const_iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &DDGGetTargetNode); + } + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &DDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits<const DataDependenceGraph *> + : public GraphTraits<const DDGNode *> { + using nodes_iterator = DataDependenceGraph::const_iterator; + static NodeRef getEntryNode(const DataDependenceGraph *DG) { + return &DG->getRoot(); + } + static nodes_iterator nodes_begin(const DataDependenceGraph *DG) { + return DG->begin(); + } + static nodes_iterator nodes_end(const DataDependenceGraph *DG) { + return DG->end(); + } +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_DDG_H diff --git a/include/llvm/Analysis/DOTGraphTraitsPass.h b/include/llvm/Analysis/DOTGraphTraitsPass.h index 0410a3314659..c9e8df5db1c2 100644 --- a/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -99,7 +99,7 @@ public: errs() << "Writing '" << Filename << "'..."; - raw_fd_ostream File(Filename, EC, sys::fs::F_Text); + raw_fd_ostream File(Filename, EC, sys::fs::OF_Text); std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); std::string Title = GraphName + " for '" + F.getName().str() + "' function"; @@ -162,7 +162,7 @@ public: errs() << "Writing '" << Filename << "'..."; - raw_fd_ostream File(Filename, EC, sys::fs::F_Text); + raw_fd_ostream File(Filename, EC, sys::fs::OF_Text); std::string Title = DOTGraphTraits<GraphT>::getGraphName(Graph); if (!EC) diff --git a/include/llvm/Analysis/DependenceGraphBuilder.h b/include/llvm/Analysis/DependenceGraphBuilder.h new file mode 100644 index 000000000000..5f4bdb47043b --- /dev/null +++ b/include/llvm/Analysis/DependenceGraphBuilder.h @@ -0,0 +1,119 @@ +//===- llvm/Analysis/DependenceGraphBuilder.h -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a builder interface that can be used to populate dependence +// graphs such as DDG and PDG. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H +#define LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H + +#include "llvm/ADT/EquivalenceClasses.h" +#include "llvm/Analysis/DependenceAnalysis.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Instructions.h" + +namespace llvm { + +/// This abstract builder class defines a set of high-level steps for creating +/// DDG-like graphs. The client code is expected to inherit from this class and +/// define concrete implementation for each of the pure virtual functions used +/// in the high-level algorithm. +template <class GraphType> class AbstractDependenceGraphBuilder { +protected: + using BasicBlockListType = SmallVectorImpl<BasicBlock *>; + +private: + using NodeType = typename GraphType::NodeType; + using EdgeType = typename GraphType::EdgeType; + +public: + using ClassesType = EquivalenceClasses<BasicBlock *>; + using NodeListType = SmallVector<NodeType *, 4>; + + AbstractDependenceGraphBuilder(GraphType &G, DependenceInfo &D, + const BasicBlockListType &BBs) + : Graph(G), DI(D), BBList(BBs) {} + virtual ~AbstractDependenceGraphBuilder() {} + + /// The main entry to the graph construction algorithm. It starts by + /// creating nodes in increasing order of granularity and then + /// adds def-use and memory edges. + /// + /// The algorithmic complexity of this implementation is O(V^2 * I^2), where V + /// is the number of vertecies (nodes) and I is the number of instructions in + /// each node. The total number of instructions, N, is equal to V * I, + /// therefore the worst-case time complexity is O(N^2). The average time + /// complexity is O((N^2)/2). + void populate() { + createFineGrainedNodes(); + createDefUseEdges(); + createMemoryDependencyEdges(); + createAndConnectRootNode(); + } + + /// Create fine grained nodes. These are typically atomic nodes that + /// consist of a single instruction. + void createFineGrainedNodes(); + + /// Analyze the def-use chains and create edges from the nodes containing + /// definitions to the nodes containing the uses. + void createDefUseEdges(); + + /// Analyze data dependencies that exist between memory loads or stores, + /// in the graph nodes and create edges between them. + void createMemoryDependencyEdges(); + + /// Create a root node and add edges such that each node in the graph is + /// reachable from the root. + void createAndConnectRootNode(); + +protected: + /// Create the root node of the graph. + virtual NodeType &createRootNode() = 0; + + /// Create an atomic node in the graph given a single instruction. + virtual NodeType &createFineGrainedNode(Instruction &I) = 0; + + /// Create a def-use edge going from \p Src to \p Tgt. + virtual EdgeType &createDefUseEdge(NodeType &Src, NodeType &Tgt) = 0; + + /// Create a memory dependence edge going from \p Src to \p Tgt. + virtual EdgeType &createMemoryEdge(NodeType &Src, NodeType &Tgt) = 0; + + /// Create a rooted edge going from \p Src to \p Tgt . + virtual EdgeType &createRootedEdge(NodeType &Src, NodeType &Tgt) = 0; + + /// Deallocate memory of edge \p E. + virtual void destroyEdge(EdgeType &E) { delete &E; } + + /// Deallocate memory of node \p N. + virtual void destroyNode(NodeType &N) { delete &N; } + + /// Map types to map instructions to nodes used when populating the graph. + using InstToNodeMap = DenseMap<Instruction *, NodeType *>; + + /// Reference to the graph that gets built by a concrete implementation of + /// this builder. + GraphType &Graph; + + /// Dependence information used to create memory dependence edges in the + /// graph. + DependenceInfo &DI; + + /// The list of basic blocks to consider when building the graph. + const BasicBlockListType &BBList; + + /// A mapping from instructions to the corresponding nodes in the graph. + InstToNodeMap IMap; +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H diff --git a/include/llvm/Analysis/DivergenceAnalysis.h b/include/llvm/Analysis/DivergenceAnalysis.h index 3cfb9d13df94..2fac9c8b4b34 100644 --- a/include/llvm/Analysis/DivergenceAnalysis.h +++ b/include/llvm/Analysis/DivergenceAnalysis.h @@ -73,9 +73,12 @@ public: /// operands bool isAlwaysUniform(const Value &Val) const; - /// \brief Whether \p Val is a divergent value + /// \brief Whether \p Val is divergent at its definition. bool isDivergent(const Value &Val) const; + /// \brief Whether \p U is divergent. Uses of a uniform value can be divergent. + bool isDivergentUse(const Use &U) const; + void print(raw_ostream &OS, const Module *) const; private: @@ -189,12 +192,19 @@ public: /// The GPU kernel this analysis result is for const Function &getFunction() const { return DA.getFunction(); } - /// Whether \p V is divergent. + /// Whether \p V is divergent at its definition. bool isDivergent(const Value &V) const; - /// Whether \p V is uniform/non-divergent + /// Whether \p U is divergent. Uses of a uniform value can be divergent. + bool isDivergentUse(const Use &U) const; + + /// Whether \p V is uniform/non-divergent. bool isUniform(const Value &V) const { return !isDivergent(V); } + /// Whether \p U is uniform/non-divergent. Uses of a uniform value can be + /// divergent. + bool isUniformUse(const Use &U) const { return !isDivergentUse(U); } + /// Print all divergent values in the kernel. void print(raw_ostream &OS, const Module *) const; }; diff --git a/include/llvm/Analysis/GlobalsModRef.h b/include/llvm/Analysis/GlobalsModRef.h index d3fcfc2d41ab..5d1c5a05206a 100644 --- a/include/llvm/Analysis/GlobalsModRef.h +++ b/include/llvm/Analysis/GlobalsModRef.h @@ -34,7 +34,7 @@ class GlobalsAAResult : public AAResultBase<GlobalsAAResult> { class FunctionInfo; const DataLayout &DL; - const TargetLibraryInfo &TLI; + std::function<const TargetLibraryInfo &(Function &F)> GetTLI; /// The globals that do not have their addresses taken. SmallPtrSet<const GlobalValue *, 8> NonAddressTakenGlobals; @@ -72,14 +72,18 @@ class GlobalsAAResult : public AAResultBase<GlobalsAAResult> { /// could perform to the memory utilization here if this becomes a problem. std::list<DeletionCallbackHandle> Handles; - explicit GlobalsAAResult(const DataLayout &DL, const TargetLibraryInfo &TLI); + explicit GlobalsAAResult( + const DataLayout &DL, + std::function<const TargetLibraryInfo &(Function &F)> GetTLI); public: GlobalsAAResult(GlobalsAAResult &&Arg); ~GlobalsAAResult(); - static GlobalsAAResult analyzeModule(Module &M, const TargetLibraryInfo &TLI, - CallGraph &CG); + static GlobalsAAResult + analyzeModule(Module &M, + std::function<const TargetLibraryInfo &(Function &F)> GetTLI, + CallGraph &CG); //------------------------------------------------ // Implement the AliasAnalysis API diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h index 054ffca7215e..a5ffca13046b 100644 --- a/include/llvm/Analysis/InstructionSimplify.h +++ b/include/llvm/Analysis/InstructionSimplify.h @@ -31,6 +31,7 @@ #ifndef LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H #define LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H +#include "llvm/ADT/SetVector.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Operator.h" #include "llvm/IR/User.h" @@ -141,6 +142,13 @@ Value *SimplifyFSubInst(Value *LHS, Value *RHS, FastMathFlags FMF, Value *SimplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q); +/// Given operands for the multiplication of a FMA, fold the result or return +/// null. In contrast to SimplifyFMulInst, this function will not perform +/// simplifications whose unrounded results differ when rounded to the argument +/// type. +Value *SimplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, + const SimplifyQuery &Q); + /// Given operands for a Mul, fold the result or return null. Value *SimplifyMulInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); @@ -234,21 +242,19 @@ Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, /// Given operand for a UnaryOperator, fold the result or return null. Value *SimplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q); -/// Given operand for an FP UnaryOperator, fold the result or return null. -/// In contrast to SimplifyUnOp, try to use FastMathFlag when folding the -/// result. In case we don't need FastMathFlags, simply fall to SimplifyUnOp. -Value *SimplifyFPUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, - const SimplifyQuery &Q); +/// Given operand for a UnaryOperator, fold the result or return null. +/// Try to use FastMathFlags when folding the result. +Value *SimplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q); /// Given operands for a BinaryOperator, fold the result or return null. Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q); -/// Given operands for an FP BinaryOperator, fold the result or return null. -/// In contrast to SimplifyBinOp, try to use FastMathFlag when folding the -/// result. In case we don't need FastMathFlags, simply fall to SimplifyBinOp. -Value *SimplifyFPBinOp(unsigned Opcode, Value *LHS, Value *RHS, - FastMathFlags FMF, const SimplifyQuery &Q); +/// Given operands for a BinaryOperator, fold the result or return null. +/// Try to use FastMathFlags when folding the result. +Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, + FastMathFlags FMF, const SimplifyQuery &Q); /// Given a callsite, fold the result or return null. Value *SimplifyCall(CallBase *Call, const SimplifyQuery &Q); @@ -263,12 +269,14 @@ Value *SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, /// This first performs a normal RAUW of I with SimpleV. It then recursively /// attempts to simplify those users updated by the operation. The 'I' /// instruction must not be equal to the simplified value 'SimpleV'. +/// If UnsimplifiedUsers is provided, instructions that could not be simplified +/// are added to it. /// /// The function returns true if any simplifications were performed. -bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, - const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr, - AssumptionCache *AC = nullptr); +bool replaceAndRecursivelySimplify( + Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI = nullptr, + const DominatorTree *DT = nullptr, AssumptionCache *AC = nullptr, + SmallSetVector<Instruction *, 8> *UnsimplifiedUsers = nullptr); /// Recursively attempt to simplify an instruction. /// diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h index 2d83929211e2..20a35bef189b 100644 --- a/include/llvm/Analysis/LazyCallGraph.h +++ b/include/llvm/Analysis/LazyCallGraph.h @@ -931,7 +931,8 @@ public: /// This sets up the graph and computes all of the entry points of the graph. /// No function definitions are scanned until their nodes in the graph are /// requested during traversal. - LazyCallGraph(Module &M, TargetLibraryInfo &TLI); + LazyCallGraph(Module &M, + function_ref<TargetLibraryInfo &(Function &)> GetTLI); LazyCallGraph(LazyCallGraph &&G); LazyCallGraph &operator=(LazyCallGraph &&RHS); @@ -1267,7 +1268,12 @@ public: /// This just builds the set of entry points to the call graph. The rest is /// built lazily as it is walked. LazyCallGraph run(Module &M, ModuleAnalysisManager &AM) { - return LazyCallGraph(M, AM.getResult<TargetLibraryAnalysis>(M)); + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { + return FAM.getResult<TargetLibraryAnalysis>(F); + }; + return LazyCallGraph(M, GetTLI); } }; diff --git a/include/llvm/Analysis/LegacyDivergenceAnalysis.h b/include/llvm/Analysis/LegacyDivergenceAnalysis.h index 0a338b816640..e33b8f4129f3 100644 --- a/include/llvm/Analysis/LegacyDivergenceAnalysis.h +++ b/include/llvm/Analysis/LegacyDivergenceAnalysis.h @@ -39,17 +39,18 @@ public: void print(raw_ostream &OS, const Module *) const override; // Returns true if V is divergent at its definition. - // - // Even if this function returns false, V may still be divergent when used - // in a different basic block. bool isDivergent(const Value *V) const; + // Returns true if U is divergent. Uses of a uniform value can be divergent. + bool isDivergentUse(const Use *U) const; + // Returns true if V is uniform/non-divergent. - // - // Even if this function returns true, V may still be divergent when used - // in a different basic block. bool isUniform(const Value *V) const { return !isDivergent(V); } + // Returns true if U is uniform/non-divergent. Uses of a uniform value can be + // divergent. + bool isUniformUse(const Use *U) const { return !isDivergentUse(U); } + // Keep the analysis results uptodate by removing an erased value. void removeValue(const Value *V) { DivergentValues.erase(V); } @@ -62,6 +63,9 @@ private: // Stores all divergent values. DenseSet<const Value *> DivergentValues; + + // Stores divergent uses of possibly uniform values. + DenseSet<const Use *> DivergentUses; }; } // End llvm namespace diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h index 5df6bb02308d..9604b2521e89 100644 --- a/include/llvm/Analysis/Loads.h +++ b/include/llvm/Analysis/Loads.h @@ -20,7 +20,9 @@ namespace llvm { class DataLayout; +class Loop; class MDNode; +class ScalarEvolution; /// Return true if this is always a dereferenceable pointer. If the context /// instruction is specified perform context-sensitive analysis and return true @@ -35,7 +37,8 @@ bool isDereferenceablePointer(const Value *V, Type *Ty, /// performs context-sensitive analysis and returns true if the pointer is /// dereferenceable at the specified instruction. bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, - unsigned Align, const DataLayout &DL, + MaybeAlign Alignment, + const DataLayout &DL, const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr); @@ -43,7 +46,7 @@ bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, /// greater or equal than requested. If the context instruction is specified /// performs context-sensitive analysis and returns true if the pointer is /// dereferenceable at the specified instruction. -bool isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, +bool isDereferenceableAndAlignedPointer(const Value *V, Align Alignment, const APInt &Size, const DataLayout &DL, const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr); @@ -56,11 +59,22 @@ bool isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, /// If it is not obviously safe to load from the specified pointer, we do a /// quick local scan of the basic block containing ScanFrom, to determine if /// the address is already accessed. -bool isSafeToLoadUnconditionally(Value *V, unsigned Align, APInt &Size, +bool isSafeToLoadUnconditionally(Value *V, MaybeAlign Alignment, APInt &Size, const DataLayout &DL, Instruction *ScanFrom = nullptr, const DominatorTree *DT = nullptr); +/// Return true if we can prove that the given load (which is assumed to be +/// within the specified loop) would access only dereferenceable memory, and +/// be properly aligned on every iteration of the specified loop regardless of +/// its placement within the loop. (i.e. does not require predication beyond +/// that required by the the header itself and could be hoisted into the header +/// if desired.) This is more powerful than the variants above when the +/// address loaded from is analyzeable by SCEV. +bool isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L, + ScalarEvolution &SE, + DominatorTree &DT); + /// Return true if we know that executing a load from this value cannot trap. /// /// If DT and ScanFrom are specified this method performs context-sensitive @@ -69,7 +83,7 @@ bool isSafeToLoadUnconditionally(Value *V, unsigned Align, APInt &Size, /// If it is not obviously safe to load from the specified pointer, we do a /// quick local scan of the basic block containing ScanFrom, to determine if /// the address is already accessed. -bool isSafeToLoadUnconditionally(Value *V, Type *Ty, unsigned Align, +bool isSafeToLoadUnconditionally(Value *V, Type *Ty, MaybeAlign Alignment, const DataLayout &DL, Instruction *ScanFrom = nullptr, const DominatorTree *DT = nullptr); diff --git a/include/llvm/Analysis/LoopAnalysisManager.h b/include/llvm/Analysis/LoopAnalysisManager.h index 368a810cfa67..a2e65a7310af 100644 --- a/include/llvm/Analysis/LoopAnalysisManager.h +++ b/include/llvm/Analysis/LoopAnalysisManager.h @@ -86,8 +86,9 @@ typedef InnerAnalysisManagerProxy<LoopAnalysisManager, Function> template <> class LoopAnalysisManagerFunctionProxy::Result { public: explicit Result(LoopAnalysisManager &InnerAM, LoopInfo &LI) - : InnerAM(&InnerAM), LI(&LI) {} - Result(Result &&Arg) : InnerAM(std::move(Arg.InnerAM)), LI(Arg.LI) { + : InnerAM(&InnerAM), LI(&LI), MSSAUsed(false) {} + Result(Result &&Arg) + : InnerAM(std::move(Arg.InnerAM)), LI(Arg.LI), MSSAUsed(Arg.MSSAUsed) { // We have to null out the analysis manager in the moved-from state // because we are taking ownership of the responsibilty to clear the // analysis state. @@ -96,6 +97,7 @@ public: Result &operator=(Result &&RHS) { InnerAM = RHS.InnerAM; LI = RHS.LI; + MSSAUsed = RHS.MSSAUsed; // We have to null out the analysis manager in the moved-from state // because we are taking ownership of the responsibilty to clear the // analysis state. @@ -112,6 +114,9 @@ public: InnerAM->clear(); } + /// Mark MemorySSA as used so we can invalidate self if MSSA is invalidated. + void markMSSAUsed() { MSSAUsed = true; } + /// Accessor for the analysis manager. LoopAnalysisManager &getManager() { return *InnerAM; } @@ -130,6 +135,7 @@ public: private: LoopAnalysisManager *InnerAM; LoopInfo *LI; + bool MSSAUsed; }; /// Provide a specialized run method for the \c LoopAnalysisManagerFunctionProxy diff --git a/include/llvm/Analysis/LoopCacheAnalysis.h b/include/llvm/Analysis/LoopCacheAnalysis.h new file mode 100644 index 000000000000..ffec78b6db2c --- /dev/null +++ b/include/llvm/Analysis/LoopCacheAnalysis.h @@ -0,0 +1,281 @@ +//===- llvm/Analysis/LoopCacheAnalysis.h ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines the interface for the loop cache analysis. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_LOOPCACHEANALYSIS_H +#define LLVM_ANALYSIS_LOOPCACHEANALYSIS_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/DependenceAnalysis.h" +#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Pass.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +class LPMUpdater; +using CacheCostTy = int64_t; +using LoopVectorTy = SmallVector<Loop *, 8>; + +/// Represents a memory reference as a base pointer and a set of indexing +/// operations. For example given the array reference A[i][2j+1][3k+2] in a +/// 3-dim loop nest: +/// for(i=0;i<n;++i) +/// for(j=0;j<m;++j) +/// for(k=0;k<o;++k) +/// ... A[i][2j+1][3k+2] ... +/// We expect: +/// BasePointer -> A +/// Subscripts -> [{0,+,1}<%for.i>][{1,+,2}<%for.j>][{2,+,3}<%for.k>] +/// Sizes -> [m][o][4] +class IndexedReference { + friend raw_ostream &operator<<(raw_ostream &OS, const IndexedReference &R); + +public: + /// Construct an indexed reference given a \p StoreOrLoadInst instruction. + IndexedReference(Instruction &StoreOrLoadInst, const LoopInfo &LI, + ScalarEvolution &SE); + + bool isValid() const { return IsValid; } + const SCEV *getBasePointer() const { return BasePointer; } + size_t getNumSubscripts() const { return Subscripts.size(); } + const SCEV *getSubscript(unsigned SubNum) const { + assert(SubNum < getNumSubscripts() && "Invalid subscript number"); + return Subscripts[SubNum]; + } + const SCEV *getFirstSubscript() const { + assert(!Subscripts.empty() && "Expecting non-empty container"); + return Subscripts.front(); + } + const SCEV *getLastSubscript() const { + assert(!Subscripts.empty() && "Expecting non-empty container"); + return Subscripts.back(); + } + + /// Return true/false if the current object and the indexed reference \p Other + /// are/aren't in the same cache line of size \p CLS. Two references are in + /// the same chace line iff the distance between them in the innermost + /// dimension is less than the cache line size. Return None if unsure. + Optional<bool> hasSpacialReuse(const IndexedReference &Other, unsigned CLS, + AliasAnalysis &AA) const; + + /// Return true if the current object and the indexed reference \p Other + /// have distance smaller than \p MaxDistance in the dimension associated with + /// the given loop \p L. Return false if the distance is not smaller than \p + /// MaxDistance and None if unsure. + Optional<bool> hasTemporalReuse(const IndexedReference &Other, + unsigned MaxDistance, const Loop &L, + DependenceInfo &DI, AliasAnalysis &AA) const; + + /// Compute the cost of the reference w.r.t. the given loop \p L when it is + /// considered in the innermost position in the loop nest. + /// The cost is defined as: + /// - equal to one if the reference is loop invariant, or + /// - equal to '(TripCount * stride) / cache_line_size' if: + /// + the reference stride is less than the cache line size, and + /// + the coefficient of this loop's index variable used in all other + /// subscripts is zero + /// - or otherwise equal to 'TripCount'. + CacheCostTy computeRefCost(const Loop &L, unsigned CLS) const; + +private: + /// Attempt to delinearize the indexed reference. + bool delinearize(const LoopInfo &LI); + + /// Return true if the index reference is invariant with respect to loop \p L. + bool isLoopInvariant(const Loop &L) const; + + /// Return true if the indexed reference is 'consecutive' in loop \p L. + /// An indexed reference is 'consecutive' if the only coefficient that uses + /// the loop induction variable is the rightmost one, and the access stride is + /// smaller than the cache line size \p CLS. + bool isConsecutive(const Loop &L, unsigned CLS) const; + + /// Return the coefficient used in the rightmost dimension. + const SCEV *getLastCoefficient() const; + + /// Return true if the coefficient corresponding to induction variable of + /// loop \p L in the given \p Subscript is zero or is loop invariant in \p L. + bool isCoeffForLoopZeroOrInvariant(const SCEV &Subscript, + const Loop &L) const; + + /// Verify that the given \p Subscript is 'well formed' (must be a simple add + /// recurrence). + bool isSimpleAddRecurrence(const SCEV &Subscript, const Loop &L) const; + + /// Return true if the given reference \p Other is definetely aliased with + /// the indexed reference represented by this class. + bool isAliased(const IndexedReference &Other, AliasAnalysis &AA) const; + +private: + /// True if the reference can be delinearized, false otherwise. + bool IsValid = false; + + /// Represent the memory reference instruction. + Instruction &StoreOrLoadInst; + + /// The base pointer of the memory reference. + const SCEV *BasePointer = nullptr; + + /// The subscript (indexes) of the memory reference. + SmallVector<const SCEV *, 3> Subscripts; + + /// The dimensions of the memory reference. + SmallVector<const SCEV *, 3> Sizes; + + ScalarEvolution &SE; +}; + +/// A reference group represents a set of memory references that exhibit +/// temporal or spacial reuse. Two references belong to the same +/// reference group with respect to a inner loop L iff: +/// 1. they have a loop independent dependency, or +/// 2. they have a loop carried dependence with a small dependence distance +/// (e.g. less than 2) carried by the inner loop, or +/// 3. they refer to the same array, and the subscript in their innermost +/// dimension is less than or equal to 'd' (where 'd' is less than the cache +/// line size) +/// +/// Intuitively a reference group represents memory references that access +/// the same cache line. Conditions 1,2 above account for temporal reuse, while +/// contition 3 accounts for spacial reuse. +using ReferenceGroupTy = SmallVector<std::unique_ptr<IndexedReference>, 8>; +using ReferenceGroupsTy = SmallVector<ReferenceGroupTy, 8>; + +/// \c CacheCost represents the estimated cost of a inner loop as the number of +/// cache lines used by the memory references it contains. +/// The 'cache cost' of a loop 'L' in a loop nest 'LN' is computed as the sum of +/// the cache costs of all of its reference groups when the loop is considered +/// to be in the innermost position in the nest. +/// A reference group represents memory references that fall into the same cache +/// line. Each reference group is analysed with respect to the innermost loop in +/// a loop nest. The cost of a reference is defined as follow: +/// - one if it is loop invariant w.r.t the innermost loop, +/// - equal to the loop trip count divided by the cache line times the +/// reference stride if the reference stride is less than the cache line +/// size (CLS), and the coefficient of this loop's index variable used in all +/// other subscripts is zero (e.g. RefCost = TripCount/(CLS/RefStride)) +/// - equal to the innermost loop trip count if the reference stride is greater +/// or equal to the cache line size CLS. +class CacheCost { + friend raw_ostream &operator<<(raw_ostream &OS, const CacheCost &CC); + using LoopTripCountTy = std::pair<const Loop *, unsigned>; + using LoopCacheCostTy = std::pair<const Loop *, CacheCostTy>; + +public: + static CacheCostTy constexpr InvalidCost = -1; + + /// Construct a CacheCost object for the loop nest described by \p Loops. + /// The optional parameter \p TRT can be used to specify the max. distance + /// between array elements accessed in a loop so that the elements are + /// classified to have temporal reuse. + CacheCost(const LoopVectorTy &Loops, const LoopInfo &LI, ScalarEvolution &SE, + TargetTransformInfo &TTI, AliasAnalysis &AA, DependenceInfo &DI, + Optional<unsigned> TRT = None); + + /// Create a CacheCost for the loop nest rooted by \p Root. + /// The optional parameter \p TRT can be used to specify the max. distance + /// between array elements accessed in a loop so that the elements are + /// classified to have temporal reuse. + static std::unique_ptr<CacheCost> + getCacheCost(Loop &Root, LoopStandardAnalysisResults &AR, DependenceInfo &DI, + Optional<unsigned> TRT = None); + + /// Return the estimated cost of loop \p L if the given loop is part of the + /// loop nest associated with this object. Return -1 otherwise. + CacheCostTy getLoopCost(const Loop &L) const { + auto IT = std::find_if( + LoopCosts.begin(), LoopCosts.end(), + [&L](const LoopCacheCostTy &LCC) { return LCC.first == &L; }); + return (IT != LoopCosts.end()) ? (*IT).second : -1; + } + + /// Return the estimated ordered loop costs. + const ArrayRef<LoopCacheCostTy> getLoopCosts() const { return LoopCosts; } + +private: + /// Calculate the cache footprint of each loop in the nest (when it is + /// considered to be in the innermost position). + void calculateCacheFootprint(); + + /// Partition store/load instructions in the loop nest into reference groups. + /// Two or more memory accesses belong in the same reference group if they + /// share the same cache line. + bool populateReferenceGroups(ReferenceGroupsTy &RefGroups) const; + + /// Calculate the cost of the given loop \p L assuming it is the innermost + /// loop in nest. + CacheCostTy computeLoopCacheCost(const Loop &L, + const ReferenceGroupsTy &RefGroups) const; + + /// Compute the cost of a representative reference in reference group \p RG + /// when the given loop \p L is considered as the innermost loop in the nest. + /// The computed cost is an estimate for the number of cache lines used by the + /// reference group. The representative reference cost is defined as: + /// - equal to one if the reference is loop invariant, or + /// - equal to '(TripCount * stride) / cache_line_size' if (a) loop \p L's + /// induction variable is used only in the reference subscript associated + /// with loop \p L, and (b) the reference stride is less than the cache + /// line size, or + /// - TripCount otherwise + CacheCostTy computeRefGroupCacheCost(const ReferenceGroupTy &RG, + const Loop &L) const; + + /// Sort the LoopCosts vector by decreasing cache cost. + void sortLoopCosts() { + sort(LoopCosts, [](const LoopCacheCostTy &A, const LoopCacheCostTy &B) { + return A.second > B.second; + }); + } + +private: + /// Loops in the loop nest associated with this object. + LoopVectorTy Loops; + + /// Trip counts for the loops in the loop nest associated with this object. + SmallVector<LoopTripCountTy, 3> TripCounts; + + /// Cache costs for the loops in the loop nest associated with this object. + SmallVector<LoopCacheCostTy, 3> LoopCosts; + + /// The max. distance between array elements accessed in a loop so that the + /// elements are classified to have temporal reuse. + Optional<unsigned> TRT; + + const LoopInfo &LI; + ScalarEvolution &SE; + TargetTransformInfo &TTI; + AliasAnalysis &AA; + DependenceInfo &DI; +}; + +raw_ostream &operator<<(raw_ostream &OS, const IndexedReference &R); +raw_ostream &operator<<(raw_ostream &OS, const CacheCost &CC); + +/// Printer pass for the \c CacheCost results. +class LoopCachePrinterPass : public PassInfoMixin<LoopCachePrinterPass> { + raw_ostream &OS; + +public: + explicit LoopCachePrinterPass(raw_ostream &OS) : OS(OS) {} + + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_LOOPCACHEANALYSIS_H diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h index 584eb3a8c854..abf3863b0601 100644 --- a/include/llvm/Analysis/LoopInfo.h +++ b/include/llvm/Analysis/LoopInfo.h @@ -30,6 +30,9 @@ // instance. In particular, a Loop might be inside such a non-loop SCC, or a // non-loop SCC might contain a sub-SCC which is a Loop. // +// For an overview of terminology used in this API (and thus all of our loop +// analyses or transforms), see docs/LoopTerminology.rst. +// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_LOOPINFO_H @@ -570,9 +573,9 @@ public: bool getIncomingAndBackEdge(BasicBlock *&Incoming, BasicBlock *&Backedge) const; - /// Below are some utilities to get loop bounds and induction variable, and - /// check if a given phinode is an auxiliary induction variable, as well as - /// checking if the loop is canonical. + /// Below are some utilities to get the loop guard, loop bounds and induction + /// variable, and to check if a given phinode is an auxiliary induction + /// variable, if the loop is guarded, and if the loop is canonical. /// /// Here is an example: /// \code @@ -604,6 +607,9 @@ public: /// /// - getInductionVariable --> i_1 /// - isAuxiliaryInductionVariable(x) --> true if x == i_1 + /// - getLoopGuardBranch() + /// --> `if (guardcmp) goto preheader; else goto afterloop` + /// - isGuarded() --> true /// - isCanonical --> false struct LoopBounds { /// Return the LoopBounds object if @@ -725,6 +731,31 @@ public: bool isAuxiliaryInductionVariable(PHINode &AuxIndVar, ScalarEvolution &SE) const; + /// Return the loop guard branch, if it exists. + /// + /// This currently only works on simplified loop, as it requires a preheader + /// and a latch to identify the guard. It will work on loops of the form: + /// \code + /// GuardBB: + /// br cond1, Preheader, ExitSucc <== GuardBranch + /// Preheader: + /// br Header + /// Header: + /// ... + /// br Latch + /// Latch: + /// br cond2, Header, ExitBlock + /// ExitBlock: + /// br ExitSucc + /// ExitSucc: + /// \endcode + BranchInst *getLoopGuardBranch() const; + + /// Return true iff the loop is + /// - in simplify rotated form, and + /// - guarded by a loop guard branch. + bool isGuarded() const { return (getLoopGuardBranch() != nullptr); } + /// Return true if the loop induction variable starts at zero and increments /// by one each time through the loop. bool isCanonical(ScalarEvolution &SE) const; diff --git a/include/llvm/Analysis/LoopInfoImpl.h b/include/llvm/Analysis/LoopInfoImpl.h index 4c33dac9e21e..8b11e848a195 100644 --- a/include/llvm/Analysis/LoopInfoImpl.h +++ b/include/llvm/Analysis/LoopInfoImpl.h @@ -85,9 +85,9 @@ template <class BlockT, class LoopT> bool LoopBase<BlockT, LoopT>::hasDedicatedExits() const { // Each predecessor of each exit block of a normal loop is contained // within the loop. - SmallVector<BlockT *, 4> ExitBlocks; - getExitBlocks(ExitBlocks); - for (BlockT *EB : ExitBlocks) + SmallVector<BlockT *, 4> UniqueExitBlocks; + getUniqueExitBlocks(UniqueExitBlocks); + for (BlockT *EB : UniqueExitBlocks) for (BlockT *Predecessor : children<Inverse<BlockT *>>(EB)) if (!contains(Predecessor)) return false; @@ -200,8 +200,6 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopPredecessor() const { } } - // Make sure there is only one exit out of the preheader. - assert(Out && "Header of loop has no predecessors from outside loop?"); return Out; } diff --git a/include/llvm/Analysis/MemoryBuiltins.h b/include/llvm/Analysis/MemoryBuiltins.h index 49f9e58ffad7..a89d76b9e5bd 100644 --- a/include/llvm/Analysis/MemoryBuiltins.h +++ b/include/llvm/Analysis/MemoryBuiltins.h @@ -58,6 +58,9 @@ class Value; /// like). bool isAllocationFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast = false); +bool isAllocationFn(const Value *V, + function_ref<const TargetLibraryInfo &(Function &)> GetTLI, + bool LookThroughBitCast = false); /// Tests if a value is a call or invoke to a function that returns a /// NoAlias pointer (including malloc/calloc/realloc/strdup-like functions). @@ -68,6 +71,9 @@ bool isNoAliasFn(const Value *V, const TargetLibraryInfo *TLI, /// allocates uninitialized memory (such as malloc). bool isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast = false); +bool isMallocLikeFn(const Value *V, + function_ref<const TargetLibraryInfo &(Function &)> GetTLI, + bool LookThroughBitCast = false); /// Tests if a value is a call or invoke to a library function that /// allocates zero-filled memory (such as calloc). @@ -93,6 +99,16 @@ bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, /// reallocates memory (e.g., realloc). bool isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI); +/// Tests if a value is a call or invoke to a library function that +/// allocates memory and throws if an allocation failed (e.g., new). +bool isOpNewLikeFn(const Value *V, const TargetLibraryInfo *TLI, + bool LookThroughBitCast = false); + +/// Tests if a value is a call or invoke to a library function that +/// allocates memory (strdup, strndup). +bool isStrdupLikeFn(const Value *V, const TargetLibraryInfo *TLI, + bool LookThroughBitCast = false); + //===----------------------------------------------------------------------===// // malloc Call Utility Functions. // @@ -100,9 +116,13 @@ bool isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI); /// extractMallocCall - Returns the corresponding CallInst if the instruction /// is a malloc call. Since CallInst::CreateMalloc() only creates calls, we /// ignore InvokeInst here. -const CallInst *extractMallocCall(const Value *I, const TargetLibraryInfo *TLI); -inline CallInst *extractMallocCall(Value *I, const TargetLibraryInfo *TLI) { - return const_cast<CallInst*>(extractMallocCall((const Value*)I, TLI)); +const CallInst * +extractMallocCall(const Value *I, + function_ref<const TargetLibraryInfo &(Function &)> GetTLI); +inline CallInst * +extractMallocCall(Value *I, + function_ref<const TargetLibraryInfo &(Function &)> GetTLI) { + return const_cast<CallInst *>(extractMallocCall((const Value *)I, GetTLI)); } /// getMallocType - Returns the PointerType resulting from the malloc call. diff --git a/include/llvm/Analysis/MemoryDependenceAnalysis.h b/include/llvm/Analysis/MemoryDependenceAnalysis.h index e2669c2fa601..e89e5690fad0 100644 --- a/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -362,11 +362,14 @@ private: PhiValues &PV; PredIteratorCache PredCache; + unsigned DefaultBlockScanLimit; + public: MemoryDependenceResults(AliasAnalysis &AA, AssumptionCache &AC, - const TargetLibraryInfo &TLI, - DominatorTree &DT, PhiValues &PV) - : AA(AA), AC(AC), TLI(TLI), DT(DT), PV(PV) {} + const TargetLibraryInfo &TLI, DominatorTree &DT, + PhiValues &PV, unsigned DefaultBlockScanLimit) + : AA(AA), AC(AC), TLI(TLI), DT(DT), PV(PV), + DefaultBlockScanLimit(DefaultBlockScanLimit) {} /// Handle invalidation in the new PM. bool invalidate(Function &F, const PreservedAnalyses &PA, @@ -511,9 +514,14 @@ class MemoryDependenceAnalysis static AnalysisKey Key; + unsigned DefaultBlockScanLimit; + public: using Result = MemoryDependenceResults; + MemoryDependenceAnalysis(); + MemoryDependenceAnalysis(unsigned DefaultBlockScanLimit) : DefaultBlockScanLimit(DefaultBlockScanLimit) { } + MemoryDependenceResults run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/include/llvm/Analysis/MemorySSA.h b/include/llvm/Analysis/MemorySSA.h index b7730be75354..e89bf26a7234 100644 --- a/include/llvm/Analysis/MemorySSA.h +++ b/include/llvm/Analysis/MemorySSA.h @@ -793,6 +793,7 @@ protected: friend class MemorySSAPrinterLegacyPass; friend class MemorySSAUpdater; + void verifyPrevDefInPhis(Function &F) const; void verifyDefUses(Function &F) const; void verifyDomination(Function &F) const; void verifyOrdering(Function &F) const; @@ -830,7 +831,8 @@ protected: void insertIntoListsBefore(MemoryAccess *, const BasicBlock *, AccessList::iterator); MemoryUseOrDef *createDefinedAccess(Instruction *, MemoryAccess *, - const MemoryUseOrDef *Template = nullptr); + const MemoryUseOrDef *Template = nullptr, + bool CreationMustSucceed = true); private: template <class AliasAnalysisType> class ClobberWalkerBase; diff --git a/include/llvm/Analysis/MemorySSAUpdater.h b/include/llvm/Analysis/MemorySSAUpdater.h index d4d8040c1ff6..1d34663721e3 100644 --- a/include/llvm/Analysis/MemorySSAUpdater.h +++ b/include/llvm/Analysis/MemorySSAUpdater.h @@ -99,7 +99,7 @@ public: /// load a /// Where a mayalias b, *does* require RenameUses be set to true. void insertDef(MemoryDef *Def, bool RenameUses = false); - void insertUse(MemoryUse *Use); + void insertUse(MemoryUse *Use, bool RenameUses = false); /// Update the MemoryPhi in `To` following an edge deletion between `From` and /// `To`. If `To` becomes unreachable, a call to removeBlocks should be made. void removeEdge(BasicBlock *From, BasicBlock *To); @@ -275,6 +275,7 @@ private: getPreviousDefRecursive(BasicBlock *, DenseMap<BasicBlock *, TrackingVH<MemoryAccess>> &); MemoryAccess *recursePhi(MemoryAccess *Phi); + MemoryAccess *tryRemoveTrivialPhi(MemoryPhi *Phi); template <class RangeType> MemoryAccess *tryRemoveTrivialPhi(MemoryPhi *Phi, RangeType &Operands); void tryRemoveTrivialPhis(ArrayRef<WeakVH> UpdatedPHIs); diff --git a/include/llvm/Analysis/MustExecute.h b/include/llvm/Analysis/MustExecute.h index 3ef539c89d97..87cf9f85c7f1 100644 --- a/include/llvm/Analysis/MustExecute.h +++ b/include/llvm/Analysis/MustExecute.h @@ -7,10 +7,17 @@ //===----------------------------------------------------------------------===// /// \file /// Contains a collection of routines for determining if a given instruction is -/// guaranteed to execute if a given point in control flow is reached. The most +/// guaranteed to execute if a given point in control flow is reached. The most /// common example is an instruction within a loop being provably executed if we /// branch to the header of it's containing loop. /// +/// There are two interfaces available to determine if an instruction is +/// executed once a given point in the control flow is reached: +/// 1) A loop-centric one derived from LoopSafetyInfo. +/// 2) A "must be executed context"-based one implemented in the +/// MustBeExecutedContextExplorer. +/// Please refer to the class comments for more information. +/// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_MUSTEXECUTE_H @@ -164,6 +171,280 @@ public: virtual ~ICFLoopSafetyInfo() {}; }; -} +struct MustBeExecutedContextExplorer; + +/// Must be executed iterators visit stretches of instructions that are +/// guaranteed to be executed together, potentially with other instruction +/// executed in-between. +/// +/// Given the following code, and assuming all statements are single +/// instructions which transfer execution to the successor (see +/// isGuaranteedToTransferExecutionToSuccessor), there are two possible +/// outcomes. If we start the iterator at A, B, or E, we will visit only A, B, +/// and E. If we start at C or D, we will visit all instructions A-E. +/// +/// \code +/// A; +/// B; +/// if (...) { +/// C; +/// D; +/// } +/// E; +/// \endcode +/// +/// +/// Below is the example extneded with instructions F and G. Now we assume F +/// might not transfer execution to it's successor G. As a result we get the +/// following visit sets: +/// +/// Start Instruction | Visit Set +/// A | A, B, E, F +/// B | A, B, E, F +/// C | A, B, C, D, E, F +/// D | A, B, C, D, E, F +/// E | A, B, E, F +/// F | A, B, E, F +/// G | A, B, E, F, G +/// +/// +/// \code +/// A; +/// B; +/// if (...) { +/// C; +/// D; +/// } +/// E; +/// F; // Might not transfer execution to its successor G. +/// G; +/// \endcode +/// +/// +/// A more complex example involving conditionals, loops, break, and continue +/// is shown below. We again assume all instructions will transmit control to +/// the successor and we assume we can prove the inner loop to be finite. We +/// omit non-trivial branch conditions as the exploration is oblivious to them. +/// Constant branches are assumed to be unconditional in the CFG. The resulting +/// visist sets are shown in the table below. +/// +/// \code +/// A; +/// while (true) { +/// B; +/// if (...) +/// C; +/// if (...) +/// continue; +/// D; +/// if (...) +/// break; +/// do { +/// if (...) +/// continue; +/// E; +/// } while (...); +/// F; +/// } +/// G; +/// \endcode +/// +/// Start Instruction | Visit Set +/// A | A, B +/// B | A, B +/// C | A, B, C +/// D | A, B, D +/// E | A, B, D, E, F +/// F | A, B, D, F +/// G | A, B, D, G +/// +/// +/// Note that the examples show optimal visist sets but not necessarily the ones +/// derived by the explorer depending on the available CFG analyses (see +/// MustBeExecutedContextExplorer). Also note that we, depending on the options, +/// the visit set can contain instructions from other functions. +struct MustBeExecutedIterator { + /// Type declarations that make his class an input iterator. + ///{ + typedef const Instruction *value_type; + typedef std::ptrdiff_t difference_type; + typedef const Instruction **pointer; + typedef const Instruction *&reference; + typedef std::input_iterator_tag iterator_category; + ///} + + using ExplorerTy = MustBeExecutedContextExplorer; + + MustBeExecutedIterator(const MustBeExecutedIterator &Other) + : Visited(Other.Visited), Explorer(Other.Explorer), + CurInst(Other.CurInst) {} + + MustBeExecutedIterator(MustBeExecutedIterator &&Other) + : Visited(std::move(Other.Visited)), Explorer(Other.Explorer), + CurInst(Other.CurInst) {} + + MustBeExecutedIterator &operator=(MustBeExecutedIterator &&Other) { + if (this != &Other) { + std::swap(Visited, Other.Visited); + std::swap(CurInst, Other.CurInst); + } + return *this; + } + + ~MustBeExecutedIterator() {} + + /// Pre- and post-increment operators. + ///{ + MustBeExecutedIterator &operator++() { + CurInst = advance(); + return *this; + } + + MustBeExecutedIterator operator++(int) { + MustBeExecutedIterator tmp(*this); + operator++(); + return tmp; + } + ///} + + /// Equality and inequality operators. Note that we ignore the history here. + ///{ + bool operator==(const MustBeExecutedIterator &Other) const { + return CurInst == Other.CurInst; + } + + bool operator!=(const MustBeExecutedIterator &Other) const { + return !(*this == Other); + } + ///} + + /// Return the underlying instruction. + const Instruction *&operator*() { return CurInst; } + const Instruction *getCurrentInst() const { return CurInst; } + + /// Return true if \p I was encountered by this iterator already. + bool count(const Instruction *I) const { return Visited.count(I); } + +private: + using VisitedSetTy = DenseSet<const Instruction *>; + + /// Private constructors. + MustBeExecutedIterator(ExplorerTy &Explorer, const Instruction *I); + + /// Reset the iterator to its initial state pointing at \p I. + void reset(const Instruction *I); + + /// Try to advance one of the underlying positions (Head or Tail). + /// + /// \return The next instruction in the must be executed context, or nullptr + /// if none was found. + const Instruction *advance(); + + /// A set to track the visited instructions in order to deal with endless + /// loops and recursion. + VisitedSetTy Visited; + + /// A reference to the explorer that created this iterator. + ExplorerTy &Explorer; + + /// The instruction we are currently exposing to the user. There is always an + /// instruction that we know is executed with the given program point, + /// initially the program point itself. + const Instruction *CurInst; + + friend struct MustBeExecutedContextExplorer; +}; + +/// A "must be executed context" for a given program point PP is the set of +/// instructions, potentially before and after PP, that are executed always when +/// PP is reached. The MustBeExecutedContextExplorer an interface to explore +/// "must be executed contexts" in a module through the use of +/// MustBeExecutedIterator. +/// +/// The explorer exposes "must be executed iterators" that traverse the must be +/// executed context. There is little information sharing between iterators as +/// the expected use case involves few iterators for "far apart" instructions. +/// If that changes, we should consider caching more intermediate results. +struct MustBeExecutedContextExplorer { + + /// In the description of the parameters we use PP to denote a program point + /// for which the must be executed context is explored, or put differently, + /// for which the MustBeExecutedIterator is created. + /// + /// \param ExploreInterBlock Flag to indicate if instructions in blocks + /// other than the parent of PP should be + /// explored. + MustBeExecutedContextExplorer(bool ExploreInterBlock) + : ExploreInterBlock(ExploreInterBlock), EndIterator(*this, nullptr) {} + + /// Clean up the dynamically allocated iterators. + ~MustBeExecutedContextExplorer() { + DeleteContainerSeconds(InstructionIteratorMap); + } + + /// Iterator-based interface. \see MustBeExecutedIterator. + ///{ + using iterator = MustBeExecutedIterator; + using const_iterator = const MustBeExecutedIterator; + + /// Return an iterator to explore the context around \p PP. + iterator &begin(const Instruction *PP) { + auto *&It = InstructionIteratorMap[PP]; + if (!It) + It = new iterator(*this, PP); + return *It; + } + + /// Return an iterator to explore the cached context around \p PP. + const_iterator &begin(const Instruction *PP) const { + return *InstructionIteratorMap.lookup(PP); + } + + /// Return an universal end iterator. + ///{ + iterator &end() { return EndIterator; } + iterator &end(const Instruction *) { return EndIterator; } + + const_iterator &end() const { return EndIterator; } + const_iterator &end(const Instruction *) const { return EndIterator; } + ///} + + /// Return an iterator range to explore the context around \p PP. + llvm::iterator_range<iterator> range(const Instruction *PP) { + return llvm::make_range(begin(PP), end(PP)); + } + + /// Return an iterator range to explore the cached context around \p PP. + llvm::iterator_range<const_iterator> range(const Instruction *PP) const { + return llvm::make_range(begin(PP), end(PP)); + } + ///} + + /// Return the next instruction that is guaranteed to be executed after \p PP. + /// + /// \param It The iterator that is used to traverse the must be + /// executed context. + /// \param PP The program point for which the next instruction + /// that is guaranteed to execute is determined. + const Instruction * + getMustBeExecutedNextInstruction(MustBeExecutedIterator &It, + const Instruction *PP); + + /// Parameter that limit the performed exploration. See the constructor for + /// their meaning. + ///{ + const bool ExploreInterBlock; + ///} + +private: + /// Map from instructions to associated must be executed iterators. + DenseMap<const Instruction *, MustBeExecutedIterator *> + InstructionIteratorMap; + + /// A unique end iterator. + MustBeExecutedIterator EndIterator; +}; + +} // namespace llvm #endif diff --git a/include/llvm/Analysis/Passes.h b/include/llvm/Analysis/Passes.h index d9c97dff8c6e..8562519fa7b1 100644 --- a/include/llvm/Analysis/Passes.h +++ b/include/llvm/Analysis/Passes.h @@ -103,6 +103,13 @@ namespace llvm { // FunctionPass *createMustExecutePrinter(); + //===--------------------------------------------------------------------===// + // + // createMustBeExecutedContextPrinter - This pass prints information about which + // instructions are guaranteed to execute together (run with -analyze). + // + ModulePass *createMustBeExecutedContextPrinter(); + } #endif diff --git a/include/llvm/Analysis/ProfileSummaryInfo.h b/include/llvm/Analysis/ProfileSummaryInfo.h index f309d344b8d1..6693e40ccf22 100644 --- a/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/include/llvm/Analysis/ProfileSummaryInfo.h @@ -52,6 +52,15 @@ private: // because the number of profile counts required to reach the hot // percentile is above a huge threshold. Optional<bool> HasHugeWorkingSetSize; + // True if the working set size of the code is considered large, + // because the number of profile counts required to reach the hot + // percentile is above a large threshold. + Optional<bool> HasLargeWorkingSetSize; + // Compute the threshold for a given cutoff. + Optional<uint64_t> computeThreshold(int PercentileCutoff); + // The map that caches the threshold values. The keys are the percentile + // cutoff values and the values are the corresponding threshold values. + DenseMap<int, uint64_t> ThresholdCache; public: ProfileSummaryInfo(Module &M) : M(M) {} @@ -96,6 +105,8 @@ public: bool AllowSynthetic = false); /// Returns true if the working set size of the code is considered huge. bool hasHugeWorkingSetSize(); + /// Returns true if the working set size of the code is considered large. + bool hasLargeWorkingSetSize(); /// Returns true if \p F has hot function entry. bool isFunctionEntryHot(const Function *F); /// Returns true if \p F contains hot code. @@ -104,14 +115,26 @@ public: bool isFunctionEntryCold(const Function *F); /// Returns true if \p F contains only cold code. bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI); + /// Returns true if \p F contains hot code with regard to a given hot + /// percentile cutoff value. + bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, + const Function *F, + BlockFrequencyInfo &BFI); /// Returns true if count \p C is considered hot. bool isHotCount(uint64_t C); /// Returns true if count \p C is considered cold. bool isColdCount(uint64_t C); + /// Returns true if count \p C is considered hot with regard to a given + /// hot percentile cutoff value. + bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C); /// Returns true if BasicBlock \p BB is considered hot. bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered cold. bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered hot with regard to a given + /// hot percentile cutoff value. + bool isHotBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if CallSite \p CS is considered hot. bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// Returns true if Callsite \p CS is considered cold. diff --git a/include/llvm/Analysis/RegionInfoImpl.h b/include/llvm/Analysis/RegionInfoImpl.h index c59c09dd2095..6b5936680c37 100644 --- a/include/llvm/Analysis/RegionInfoImpl.h +++ b/include/llvm/Analysis/RegionInfoImpl.h @@ -365,7 +365,7 @@ typename Tr::RegionNodeT *RegionBase<Tr>::getBBNode(BlockT *BB) const { auto Deconst = const_cast<RegionBase<Tr> *>(this); typename BBNodeMapT::value_type V = { BB, - llvm::make_unique<RegionNodeT>(static_cast<RegionT *>(Deconst), BB)}; + std::make_unique<RegionNodeT>(static_cast<RegionT *>(Deconst), BB)}; at = BBNodeMap.insert(std::move(V)).first; } return at->second.get(); diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index 0bd98ef37e7a..9c55f7a5090f 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -468,6 +468,8 @@ template <> struct DenseMapInfo<ExitLimitQuery> { /// can't do much with the SCEV objects directly, they must ask this class /// for services. class ScalarEvolution { + friend class ScalarEvolutionsTest; + public: /// An enum describing the relationship between a SCEV and a loop. enum LoopDisposition { @@ -777,10 +779,10 @@ public: /// to (i.e. a "conservative over-approximation") of the value returend by /// getBackedgeTakenCount. If such a value cannot be computed, it returns the /// SCEVCouldNotCompute object. - const SCEV *getMaxBackedgeTakenCount(const Loop *L); + const SCEV *getConstantMaxBackedgeTakenCount(const Loop *L); /// Return true if the backedge taken count is either the value returned by - /// getMaxBackedgeTakenCount or zero. + /// getConstantMaxBackedgeTakenCount or zero. bool isBackedgeTakenCountMaxOrZero(const Loop *L); /// Return true if the specified loop has an analyzable loop-invariant diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h index a519f93216b3..b4d727449fbe 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -77,9 +77,13 @@ namespace llvm { /// Phis that complete an IV chain. Reuse DenseSet<AssertingVH<PHINode>> ChainedPhis; - /// When true, expressions are expanded in "canonical" form. In particular, - /// addrecs are expanded as arithmetic based on a canonical induction - /// variable. When false, expression are expanded in a more literal form. + /// When true, SCEVExpander tries to expand expressions in "canonical" form. + /// When false, expressions are expanded in a more literal form. + /// + /// In "canonical" form addrecs are expanded as arithmetic based on a + /// canonical induction variable. Note that CanonicalMode doesn't guarantee + /// that all expressions are expanded in "canonical" form. For some + /// expressions literal mode can be preferred. bool CanonicalMode; /// When invoked from LSR, the expander is in "strength reduction" mode. The @@ -275,8 +279,16 @@ namespace llvm { /// Clear the current insertion point. This is useful if the instruction /// that had been serving as the insertion point may have been deleted. - void clearInsertPoint() { - Builder.ClearInsertionPoint(); + void clearInsertPoint() { Builder.ClearInsertionPoint(); } + + /// Set location information used by debugging information. + void SetCurrentDebugLocation(DebugLoc L) { + Builder.SetCurrentDebugLocation(std::move(L)); + } + + /// Get location information used by debugging information. + const DebugLoc &getCurrentDebugLocation() const { + return Builder.getCurrentDebugLocation(); } /// Return true if the specified instruction was inserted by the code diff --git a/include/llvm/Analysis/TargetLibraryInfo.h b/include/llvm/Analysis/TargetLibraryInfo.h index 4b5200f5a838..d4b223863c54 100644 --- a/include/llvm/Analysis/TargetLibraryInfo.h +++ b/include/llvm/Analysis/TargetLibraryInfo.h @@ -30,11 +30,12 @@ struct VecDesc { unsigned VectorizationFactor; }; - enum LibFunc { + enum LibFunc : unsigned { #define TLI_DEFINE_ENUM #include "llvm/Analysis/TargetLibraryInfo.def" - NumLibFuncs + NumLibFuncs, + NotLibFunc }; /// Implementation of the target library information. @@ -48,7 +49,7 @@ class TargetLibraryInfoImpl { unsigned char AvailableArray[(NumLibFuncs+3)/4]; llvm::DenseMap<unsigned, std::string> CustomNames; - static StringRef const StandardNames[NumLibFuncs]; + static StringLiteral const StandardNames[NumLibFuncs]; bool ShouldExtI32Param, ShouldExtI32Return, ShouldSignExtI32Param; enum AvailabilityState { @@ -359,7 +360,6 @@ public: TargetLibraryAnalysis(TargetLibraryInfoImpl PresetInfoImpl) : PresetInfoImpl(std::move(PresetInfoImpl)) {} - TargetLibraryInfo run(Module &M, ModuleAnalysisManager &); TargetLibraryInfo run(Function &F, FunctionAnalysisManager &); private: @@ -385,8 +385,13 @@ public: explicit TargetLibraryInfoWrapperPass(const Triple &T); explicit TargetLibraryInfoWrapperPass(const TargetLibraryInfoImpl &TLI); - TargetLibraryInfo &getTLI() { return TLI; } - const TargetLibraryInfo &getTLI() const { return TLI; } + TargetLibraryInfo &getTLI(const Function &F LLVM_ATTRIBUTE_UNUSED) { + return TLI; + } + const TargetLibraryInfo & + getTLI(const Function &F LLVM_ATTRIBUTE_UNUSED) const { + return TLI; + } }; } // end namespace llvm diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h index 7574b811bc1c..d6fa88411654 100644 --- a/include/llvm/Analysis/TargetTransformInfo.h +++ b/include/llvm/Analysis/TargetTransformInfo.h @@ -368,6 +368,20 @@ public: /// optimize away. unsigned getFlatAddressSpace() const; + /// Return any intrinsic address operand indexes which may be rewritten if + /// they use a flat address space pointer. + /// + /// \returns true if the intrinsic was handled. + bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, + Intrinsic::ID IID) const; + + /// Rewrite intrinsic call \p II such that \p OldV will be replaced with \p + /// NewV, which has a different address space. This should happen for every + /// operand index that collectFlatAddressOperands returned for the intrinsic. + /// \returns true if the intrinsic /// was handled. + bool rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, + Value *OldV, Value *NewV) const; + /// Test whether calls to a function lower to actual program function /// calls. /// @@ -469,12 +483,17 @@ public: bool Force; /// Allow using trip count upper bound to unroll loops. bool UpperBound; - /// Allow peeling off loop iterations for loops with low dynamic tripcount. + /// Allow peeling off loop iterations. bool AllowPeeling; /// Allow unrolling of all the iterations of the runtime loop remainder. bool UnrollRemainder; /// Allow unroll and jam. Used to enable unroll and jam for the target. bool UnrollAndJam; + /// Allow peeling basing on profile. Uses to enable peeling off all + /// iterations basing on provided profile. + /// If the value is true the peeling cost model can decide to peel only + /// some iterations and in this case it will set this to false. + bool PeelProfiledIterations; /// Threshold for unroll and jam, for inner loop size. The 'Threshold' /// value above is used during unroll and jam for the outer loop size. /// This value is used in the same manner to limit the size of the inner @@ -555,15 +574,15 @@ public: /// modes that operate across loop iterations. bool shouldFavorBackedgeIndex(const Loop *L) const; - /// Return true if the target supports masked load. - bool isLegalMaskedStore(Type *DataType) const; /// Return true if the target supports masked store. - bool isLegalMaskedLoad(Type *DataType) const; + bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) const; + /// Return true if the target supports masked load. + bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) const; /// Return true if the target supports nontemporal store. - bool isLegalNTStore(Type *DataType, unsigned Alignment) const; + bool isLegalNTStore(Type *DataType, Align Alignment) const; /// Return true if the target supports nontemporal load. - bool isLegalNTLoad(Type *DataType, unsigned Alignment) const; + bool isLegalNTLoad(Type *DataType, Align Alignment) const; /// Return true if the target supports masked scatter. bool isLegalMaskedScatter(Type *DataType) const; @@ -622,12 +641,6 @@ public: /// Return true if this type is legal. bool isTypeLegal(Type *Ty) const; - /// Returns the target's jmp_buf alignment in bytes. - unsigned getJumpBufAlignment() const; - - /// Returns the target's jmp_buf size in bytes. - unsigned getJumpBufSize() const; - /// Return true if switches should be turned into lookup tables for the /// target. bool shouldBuildLookupTables() const; @@ -775,10 +788,23 @@ public: /// Additional properties of an operand's values. enum OperandValueProperties { OP_None = 0, OP_PowerOf2 = 1 }; - /// \return The number of scalar or vector registers that the target has. - /// If 'Vectors' is true, it returns the number of vector registers. If it is - /// set to false, it returns the number of scalar registers. - unsigned getNumberOfRegisters(bool Vector) const; + /// \return the number of registers in the target-provided register class. + unsigned getNumberOfRegisters(unsigned ClassID) const; + + /// \return the target-provided register class ID for the provided type, + /// accounting for type promotion and other type-legalization techniques that the target might apply. + /// However, it specifically does not account for the scalarization or splitting of vector types. + /// Should a vector type require scalarization or splitting into multiple underlying vector registers, + /// that type should be mapped to a register class containing no registers. + /// Specifically, this is designed to provide a simple, high-level view of the register allocation + /// later performed by the backend. These register classes don't necessarily map onto the + /// register classes used by the backend. + /// FIXME: It's not currently possible to determine how many registers + /// are used by the provided type. + unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const; + + /// \return the target-provided register class name + const char* getRegisterClassName(unsigned ClassID) const; /// \return The width of the largest scalar or vector register type. unsigned getRegisterBitWidth(bool Vector) const; @@ -824,18 +850,20 @@ public: /// \return The associativity of the cache level, if available. llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) const; - /// \return How much before a load we should place the prefetch instruction. - /// This is currently measured in number of instructions. + /// \return How much before a load we should place the prefetch + /// instruction. This is currently measured in number of + /// instructions. unsigned getPrefetchDistance() const; - /// \return Some HW prefetchers can handle accesses up to a certain constant - /// stride. This is the minimum stride in bytes where it makes sense to start - /// adding SW prefetches. The default is 1, i.e. prefetch with any stride. + /// \return Some HW prefetchers can handle accesses up to a certain + /// constant stride. This is the minimum stride in bytes where it + /// makes sense to start adding SW prefetches. The default is 1, + /// i.e. prefetch with any stride. unsigned getMinPrefetchStride() const; - /// \return The maximum number of iterations to prefetch ahead. If the - /// required number of iterations is more than this number, no prefetching is - /// performed. + /// \return The maximum number of iterations to prefetch ahead. If + /// the required number of iterations is more than this number, no + /// prefetching is performed. unsigned getMaxPrefetchIterationsAhead() const; /// \return The maximum interleave factor that any transform should try to @@ -1155,6 +1183,10 @@ public: virtual bool isSourceOfDivergence(const Value *V) = 0; virtual bool isAlwaysUniform(const Value *V) = 0; virtual unsigned getFlatAddressSpace() = 0; + virtual bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, + Intrinsic::ID IID) const = 0; + virtual bool rewriteIntrinsicWithAddressSpace( + IntrinsicInst *II, Value *OldV, Value *NewV) const = 0; virtual bool isLoweredToCall(const Function *F) = 0; virtual void getUnrollingPreferences(Loop *L, ScalarEvolution &, UnrollingPreferences &UP) = 0; @@ -1177,10 +1209,10 @@ public: TargetLibraryInfo *LibInfo) = 0; virtual bool shouldFavorPostInc() const = 0; virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0; - virtual bool isLegalMaskedStore(Type *DataType) = 0; - virtual bool isLegalMaskedLoad(Type *DataType) = 0; - virtual bool isLegalNTStore(Type *DataType, unsigned Alignment) = 0; - virtual bool isLegalNTLoad(Type *DataType, unsigned Alignment) = 0; + virtual bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) = 0; + virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0; + virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0; + virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; virtual bool isLegalMaskedScatter(Type *DataType) = 0; virtual bool isLegalMaskedGather(Type *DataType) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; @@ -1196,8 +1228,6 @@ public: virtual bool isProfitableToHoist(Instruction *I) = 0; virtual bool useAA() = 0; virtual bool isTypeLegal(Type *Ty) = 0; - virtual unsigned getJumpBufAlignment() = 0; - virtual unsigned getJumpBufSize() = 0; virtual bool shouldBuildLookupTables() = 0; virtual bool shouldBuildLookupTablesForConstant(Constant *C) = 0; virtual bool useColdCCForColdCall(Function &F) = 0; @@ -1228,19 +1258,35 @@ public: Type *Ty) = 0; virtual int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty) = 0; - virtual unsigned getNumberOfRegisters(bool Vector) = 0; + virtual unsigned getNumberOfRegisters(unsigned ClassID) const = 0; + virtual unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const = 0; + virtual const char* getRegisterClassName(unsigned ClassID) const = 0; virtual unsigned getRegisterBitWidth(bool Vector) const = 0; virtual unsigned getMinVectorRegisterBitWidth() = 0; virtual bool shouldMaximizeVectorBandwidth(bool OptSize) const = 0; virtual unsigned getMinimumVF(unsigned ElemWidth) const = 0; virtual bool shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) = 0; - virtual unsigned getCacheLineSize() = 0; - virtual llvm::Optional<unsigned> getCacheSize(CacheLevel Level) = 0; - virtual llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) = 0; - virtual unsigned getPrefetchDistance() = 0; - virtual unsigned getMinPrefetchStride() = 0; - virtual unsigned getMaxPrefetchIterationsAhead() = 0; + virtual unsigned getCacheLineSize() const = 0; + virtual llvm::Optional<unsigned> getCacheSize(CacheLevel Level) const = 0; + virtual llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) const = 0; + + /// \return How much before a load we should place the prefetch + /// instruction. This is currently measured in number of + /// instructions. + virtual unsigned getPrefetchDistance() const = 0; + + /// \return Some HW prefetchers can handle accesses up to a certain + /// constant stride. This is the minimum stride in bytes where it + /// makes sense to start adding SW prefetches. The default is 1, + /// i.e. prefetch with any stride. + virtual unsigned getMinPrefetchStride() const = 0; + + /// \return The maximum number of iterations to prefetch ahead. If + /// the required number of iterations is more than this number, no + /// prefetching is performed. + virtual unsigned getMaxPrefetchIterationsAhead() const = 0; + virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0; virtual unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, @@ -1395,6 +1441,16 @@ public: return Impl.getFlatAddressSpace(); } + bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, + Intrinsic::ID IID) const override { + return Impl.collectFlatAddressOperands(OpIndexes, IID); + } + + bool rewriteIntrinsicWithAddressSpace( + IntrinsicInst *II, Value *OldV, Value *NewV) const override { + return Impl.rewriteIntrinsicWithAddressSpace(II, OldV, NewV); + } + bool isLoweredToCall(const Function *F) override { return Impl.isLoweredToCall(F); } @@ -1440,16 +1496,16 @@ public: bool shouldFavorBackedgeIndex(const Loop *L) const override { return Impl.shouldFavorBackedgeIndex(L); } - bool isLegalMaskedStore(Type *DataType) override { - return Impl.isLegalMaskedStore(DataType); + bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedStore(DataType, Alignment); } - bool isLegalMaskedLoad(Type *DataType) override { - return Impl.isLegalMaskedLoad(DataType); + bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedLoad(DataType, Alignment); } - bool isLegalNTStore(Type *DataType, unsigned Alignment) override { + bool isLegalNTStore(Type *DataType, Align Alignment) override { return Impl.isLegalNTStore(DataType, Alignment); } - bool isLegalNTLoad(Type *DataType, unsigned Alignment) override { + bool isLegalNTLoad(Type *DataType, Align Alignment) override { return Impl.isLegalNTLoad(DataType, Alignment); } bool isLegalMaskedScatter(Type *DataType) override { @@ -1490,8 +1546,6 @@ public: } bool useAA() override { return Impl.useAA(); } bool isTypeLegal(Type *Ty) override { return Impl.isTypeLegal(Ty); } - unsigned getJumpBufAlignment() override { return Impl.getJumpBufAlignment(); } - unsigned getJumpBufSize() override { return Impl.getJumpBufSize(); } bool shouldBuildLookupTables() override { return Impl.shouldBuildLookupTables(); } @@ -1563,8 +1617,14 @@ public: Type *Ty) override { return Impl.getIntImmCost(IID, Idx, Imm, Ty); } - unsigned getNumberOfRegisters(bool Vector) override { - return Impl.getNumberOfRegisters(Vector); + unsigned getNumberOfRegisters(unsigned ClassID) const override { + return Impl.getNumberOfRegisters(ClassID); + } + unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const override { + return Impl.getRegisterClassForType(Vector, Ty); + } + const char* getRegisterClassName(unsigned ClassID) const override { + return Impl.getRegisterClassName(ClassID); } unsigned getRegisterBitWidth(bool Vector) const override { return Impl.getRegisterBitWidth(Vector); @@ -1583,22 +1643,36 @@ public: return Impl.shouldConsiderAddressTypePromotion( I, AllowPromotionWithoutCommonHeader); } - unsigned getCacheLineSize() override { + unsigned getCacheLineSize() const override { return Impl.getCacheLineSize(); } - llvm::Optional<unsigned> getCacheSize(CacheLevel Level) override { + llvm::Optional<unsigned> getCacheSize(CacheLevel Level) const override { return Impl.getCacheSize(Level); } - llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) override { + llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) const override { return Impl.getCacheAssociativity(Level); } - unsigned getPrefetchDistance() override { return Impl.getPrefetchDistance(); } - unsigned getMinPrefetchStride() override { + + /// Return the preferred prefetch distance in terms of instructions. + /// + unsigned getPrefetchDistance() const override { + return Impl.getPrefetchDistance(); + } + + /// Return the minimum stride necessary to trigger software + /// prefetching. + /// + unsigned getMinPrefetchStride() const override { return Impl.getMinPrefetchStride(); } - unsigned getMaxPrefetchIterationsAhead() override { + + /// Return the maximum prefetch distance in terms of loop + /// iterations. + /// + unsigned getMaxPrefetchIterationsAhead() const override { return Impl.getMaxPrefetchIterationsAhead(); } + unsigned getMaxInterleaveFactor(unsigned VF) override { return Impl.getMaxInterleaveFactor(VF); } diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h index b99e1eb9adf0..a431fa0d458b 100644 --- a/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -156,6 +156,16 @@ public: return -1; } + bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, + Intrinsic::ID IID) const { + return false; + } + + bool rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, + Value *OldV, Value *NewV) const { + return false; + } + bool isLoweredToCall(const Function *F) { assert(F && "A concrete function must be provided to this routine."); @@ -233,18 +243,18 @@ public: bool shouldFavorBackedgeIndex(const Loop *L) const { return false; } - bool isLegalMaskedStore(Type *DataType) { return false; } + bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) { return false; } - bool isLegalMaskedLoad(Type *DataType) { return false; } + bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) { return false; } - bool isLegalNTStore(Type *DataType, unsigned Alignment) { + bool isLegalNTStore(Type *DataType, Align Alignment) { // By default, assume nontemporal memory stores are available for stores // that are aligned and have a size that is a power of 2. unsigned DataSize = DL.getTypeStoreSize(DataType); return Alignment >= DataSize && isPowerOf2_32(DataSize); } - bool isLegalNTLoad(Type *DataType, unsigned Alignment) { + bool isLegalNTLoad(Type *DataType, Align Alignment) { // By default, assume nontemporal memory loads are available for loads that // are aligned and have a size that is a power of 2. unsigned DataSize = DL.getTypeStoreSize(DataType); @@ -284,10 +294,6 @@ public: bool isTypeLegal(Type *Ty) { return false; } - unsigned getJumpBufAlignment() { return 0; } - - unsigned getJumpBufSize() { return 0; } - bool shouldBuildLookupTables() { return true; } bool shouldBuildLookupTablesForConstant(Constant *C) { return true; } @@ -348,7 +354,20 @@ public: return TTI::TCC_Free; } - unsigned getNumberOfRegisters(bool Vector) { return 8; } + unsigned getNumberOfRegisters(unsigned ClassID) const { return 8; } + + unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const { + return Vector ? 1 : 0; + }; + + const char* getRegisterClassName(unsigned ClassID) const { + switch (ClassID) { + default: + return "Generic::Unknown Register Class"; + case 0: return "Generic::ScalarRC"; + case 1: return "Generic::VectorRC"; + } + } unsigned getRegisterBitWidth(bool Vector) const { return 32; } @@ -365,21 +384,20 @@ public: return false; } - unsigned getCacheLineSize() { return 0; } + unsigned getCacheLineSize() const { return 0; } - llvm::Optional<unsigned> getCacheSize(TargetTransformInfo::CacheLevel Level) { + llvm::Optional<unsigned> getCacheSize(TargetTransformInfo::CacheLevel Level) const { switch (Level) { case TargetTransformInfo::CacheLevel::L1D: LLVM_FALLTHROUGH; case TargetTransformInfo::CacheLevel::L2D: return llvm::Optional<unsigned>(); } - llvm_unreachable("Unknown TargetTransformInfo::CacheLevel"); } llvm::Optional<unsigned> getCacheAssociativity( - TargetTransformInfo::CacheLevel Level) { + TargetTransformInfo::CacheLevel Level) const { switch (Level) { case TargetTransformInfo::CacheLevel::L1D: LLVM_FALLTHROUGH; @@ -390,11 +408,9 @@ public: llvm_unreachable("Unknown TargetTransformInfo::CacheLevel"); } - unsigned getPrefetchDistance() { return 0; } - - unsigned getMinPrefetchStride() { return 1; } - - unsigned getMaxPrefetchIterationsAhead() { return UINT_MAX; } + unsigned getPrefetchDistance() const { return 0; } + unsigned getMinPrefetchStride() const { return 1; } + unsigned getMaxPrefetchIterationsAhead() const { return UINT_MAX; } unsigned getMaxInterleaveFactor(unsigned VF) { return 1; } @@ -830,6 +846,9 @@ public: if (isa<PHINode>(U)) return TTI::TCC_Free; // Model all PHI nodes as free. + if (isa<ExtractValueInst>(U)) + return TTI::TCC_Free; // Model all ExtractValue nodes as free. + // Static alloca doesn't generate target instructions. if (auto *A = dyn_cast<AllocaInst>(U)) if (A->isStaticAlloca()) diff --git a/include/llvm/Analysis/TypeMetadataUtils.h b/include/llvm/Analysis/TypeMetadataUtils.h index 82cf8efeea54..43ce26147c2e 100644 --- a/include/llvm/Analysis/TypeMetadataUtils.h +++ b/include/llvm/Analysis/TypeMetadataUtils.h @@ -50,6 +50,8 @@ void findDevirtualizableCallsForTypeCheckedLoad( SmallVectorImpl<Instruction *> &LoadedPtrs, SmallVectorImpl<Instruction *> &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT); + +Constant *getPointerAtOffset(Constant *I, uint64_t Offset, Module &M); } #endif diff --git a/include/llvm/Analysis/Utils/Local.h b/include/llvm/Analysis/Utils/Local.h index acbdf5dca32c..a63bcec9bc41 100644 --- a/include/llvm/Analysis/Utils/Local.h +++ b/include/llvm/Analysis/Utils/Local.h @@ -32,7 +32,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, Value *Result = Constant::getNullValue(IntPtrTy); // If the GEP is inbounds, we know that none of the addressing operations will - // overflow in an unsigned sense. + // overflow in a signed sense. bool isInBounds = GEPOp->isInBounds() && !NoAssumptions; // Build a mask for high order bits. @@ -51,10 +51,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, // Handle a struct index, which adds its field offset to the pointer. if (StructType *STy = GTI.getStructTypeOrNull()) { - if (OpC->getType()->isVectorTy()) - OpC = OpC->getSplatValue(); - - uint64_t OpValue = cast<ConstantInt>(OpC)->getZExtValue(); + uint64_t OpValue = OpC->getUniqueInteger().getZExtValue(); Size = DL.getStructLayout(STy)->getElementOffset(OpValue); if (Size) @@ -63,20 +60,31 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, continue; } + // Splat the constant if needed. + if (IntPtrTy->isVectorTy() && !OpC->getType()->isVectorTy()) + OpC = ConstantVector::getSplat(IntPtrTy->getVectorNumElements(), OpC); + Constant *Scale = ConstantInt::get(IntPtrTy, Size); Constant *OC = ConstantExpr::getIntegerCast(OpC, IntPtrTy, true /*SExt*/); - Scale = ConstantExpr::getMul(OC, Scale, isInBounds/*NUW*/); + Scale = + ConstantExpr::getMul(OC, Scale, false /*NUW*/, isInBounds /*NSW*/); // Emit an add instruction. Result = Builder->CreateAdd(Result, Scale, GEP->getName()+".offs"); continue; } + + // Splat the index if needed. + if (IntPtrTy->isVectorTy() && !Op->getType()->isVectorTy()) + Op = Builder->CreateVectorSplat(IntPtrTy->getVectorNumElements(), Op); + // Convert to correct type. if (Op->getType() != IntPtrTy) Op = Builder->CreateIntCast(Op, IntPtrTy, true, Op->getName()+".c"); if (Size != 1) { // We'll let instcombine(mul) convert this to a shl if possible. Op = Builder->CreateMul(Op, ConstantInt::get(IntPtrTy, Size), - GEP->getName()+".idx", isInBounds /*NUW*/); + GEP->getName() + ".idx", false /*NUW*/, + isInBounds /*NSW*/); } // Emit an add instruction. diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h index fa7e0e0eef7e..33b064fcf9d2 100644 --- a/include/llvm/Analysis/ValueTracking.h +++ b/include/llvm/Analysis/ValueTracking.h @@ -242,19 +242,21 @@ class Value; /// This is a wrapper around Value::stripAndAccumulateConstantOffsets that /// creates and later unpacks the required APInt. inline Value *GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, - const DataLayout &DL) { + const DataLayout &DL, + bool AllowNonInbounds = true) { APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); Value *Base = - Ptr->stripAndAccumulateConstantOffsets(DL, OffsetAPInt, - /* AllowNonInbounds */ true); + Ptr->stripAndAccumulateConstantOffsets(DL, OffsetAPInt, AllowNonInbounds); + Offset = OffsetAPInt.getSExtValue(); return Base; } - inline const Value *GetPointerBaseWithConstantOffset(const Value *Ptr, - int64_t &Offset, - const DataLayout &DL) { - return GetPointerBaseWithConstantOffset(const_cast<Value *>(Ptr), Offset, - DL); + inline const Value * + GetPointerBaseWithConstantOffset(const Value *Ptr, int64_t &Offset, + const DataLayout &DL, + bool AllowNonInbounds = true) { + return GetPointerBaseWithConstantOffset(const_cast<Value *>(Ptr), Offset, DL, + AllowNonInbounds); } /// Returns true if the GEP is based on a pointer to a string (array of @@ -307,20 +309,26 @@ class Value; uint64_t GetStringLength(const Value *V, unsigned CharSize = 8); /// This function returns call pointer argument that is considered the same by - /// aliasing rules. You CAN'T use it to replace one value with another. - const Value *getArgumentAliasingToReturnedPointer(const CallBase *Call); - inline Value *getArgumentAliasingToReturnedPointer(CallBase *Call) { + /// aliasing rules. You CAN'T use it to replace one value with another. If + /// \p MustPreserveNullness is true, the call must preserve the nullness of + /// the pointer. + const Value *getArgumentAliasingToReturnedPointer(const CallBase *Call, + bool MustPreserveNullness); + inline Value * + getArgumentAliasingToReturnedPointer(CallBase *Call, + bool MustPreserveNullness) { return const_cast<Value *>(getArgumentAliasingToReturnedPointer( - const_cast<const CallBase *>(Call))); + const_cast<const CallBase *>(Call), MustPreserveNullness)); } - // {launder,strip}.invariant.group returns pointer that aliases its argument, - // and it only captures pointer by returning it. - // These intrinsics are not marked as nocapture, because returning is - // considered as capture. The arguments are not marked as returned neither, - // because it would make it useless. + /// {launder,strip}.invariant.group returns pointer that aliases its argument, + /// and it only captures pointer by returning it. + /// These intrinsics are not marked as nocapture, because returning is + /// considered as capture. The arguments are not marked as returned neither, + /// because it would make it useless. If \p MustPreserveNullness is true, + /// the intrinsic must preserve the nullness of the pointer. bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing( - const CallBase *Call); + const CallBase *Call, bool MustPreserveNullness); /// This method strips off any GEP address adjustments and pointer casts from /// the specified value, returning the original object being addressed. Note @@ -376,6 +384,13 @@ class Value; /// Return true if the only users of this pointer are lifetime markers. bool onlyUsedByLifetimeMarkers(const Value *V); + /// Return true if speculation of the given load must be suppressed to avoid + /// ordering or interfering with an active sanitizer. If not suppressed, + /// dereferenceability and alignment must be proven separately. Note: This + /// is only needed for raw reasoning; if you use the interface below + /// (isSafeToSpeculativelyExecute), this is handled internally. + bool mustSuppressSpeculation(const LoadInst &LI); + /// Return true if the instruction does not have any effects besides /// calculating the result and does not have undefined behavior. /// @@ -605,12 +620,12 @@ class Value; SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp = nullptr, unsigned Depth = 0); + inline SelectPatternResult - matchSelectPattern(const Value *V, const Value *&LHS, const Value *&RHS, - Instruction::CastOps *CastOp = nullptr) { - Value *L = const_cast<Value*>(LHS); - Value *R = const_cast<Value*>(RHS); - auto Result = matchSelectPattern(const_cast<Value*>(V), L, R); + matchSelectPattern(const Value *V, const Value *&LHS, const Value *&RHS) { + Value *L = const_cast<Value *>(LHS); + Value *R = const_cast<Value *>(RHS); + auto Result = matchSelectPattern(const_cast<Value *>(V), L, R); LHS = L; RHS = R; return Result; @@ -654,6 +669,12 @@ class Value; Optional<bool> isImpliedByDomCondition(const Value *Cond, const Instruction *ContextI, const DataLayout &DL); + + /// If Ptr1 is provably equal to Ptr2 plus a constant offset, return that + /// offset. For example, Ptr1 might be &A[42], and Ptr2 might be &A[40]. In + /// this case offset would be -8. + Optional<int64_t> isPointerOffset(const Value *Ptr1, const Value *Ptr2, + const DataLayout &DL); } // end namespace llvm #endif // LLVM_ANALYSIS_VALUETRACKING_H diff --git a/include/llvm/Analysis/VectorUtils.h b/include/llvm/Analysis/VectorUtils.h index d93d2bc4570b..4a61c2bc35c7 100644 --- a/include/llvm/Analysis/VectorUtils.h +++ b/include/llvm/Analysis/VectorUtils.h @@ -15,18 +15,129 @@ #include "llvm/ADT/MapVector.h" #include "llvm/Analysis/LoopAccessAnalysis.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" namespace llvm { +/// Describes the type of Parameters +enum class VFParamKind { + Vector, // No semantic information. + OMP_Linear, // declare simd linear(i) + OMP_LinearRef, // declare simd linear(ref(i)) + OMP_LinearVal, // declare simd linear(val(i)) + OMP_LinearUVal, // declare simd linear(uval(i)) + OMP_LinearPos, // declare simd linear(i:c) uniform(c) + OMP_LinearValPos, // declare simd linear(val(i:c)) uniform(c) + OMP_LinearRefPos, // declare simd linear(ref(i:c)) uniform(c) + OMP_LinearUValPos, // declare simd linear(uval(i:c)) uniform(c + OMP_Uniform, // declare simd uniform(i) + GlobalPredicate, // Global logical predicate that acts on all lanes + // of the input and output mask concurrently. For + // example, it is implied by the `M` token in the + // Vector Function ABI mangled name. + Unknown +}; + +/// Describes the type of Instruction Set Architecture +enum class VFISAKind { + AdvancedSIMD, // AArch64 Advanced SIMD (NEON) + SVE, // AArch64 Scalable Vector Extension + SSE, // x86 SSE + AVX, // x86 AVX + AVX2, // x86 AVX2 + AVX512, // x86 AVX512 + Unknown // Unknown ISA +}; + +/// Encapsulates information needed to describe a parameter. +/// +/// The description of the parameter is not linked directly to +/// OpenMP or any other vector function description. This structure +/// is extendible to handle other paradigms that describe vector +/// functions and their parameters. +struct VFParameter { + unsigned ParamPos; // Parameter Position in Scalar Function. + VFParamKind ParamKind; // Kind of Parameter. + int LinearStepOrPos = 0; // Step or Position of the Parameter. + Align Alignment = Align(); // Optional aligment in bytes, defaulted to 1. + + // Comparison operator. + bool operator==(const VFParameter &Other) const { + return std::tie(ParamPos, ParamKind, LinearStepOrPos, Alignment) == + std::tie(Other.ParamPos, Other.ParamKind, Other.LinearStepOrPos, + Other.Alignment); + } +}; + +/// Contains the information about the kind of vectorization +/// available. +/// +/// This object in independent on the paradigm used to +/// represent vector functions. in particular, it is not attached to +/// any target-specific ABI. +struct VFShape { + unsigned VF; // Vectorization factor. + bool IsScalable; // True if the function is a scalable function. + VFISAKind ISA; // Instruction Set Architecture. + SmallVector<VFParameter, 8> Parameters; // List of parameter informations. + // Comparison operator. + bool operator==(const VFShape &Other) const { + return std::tie(VF, IsScalable, ISA, Parameters) == + std::tie(Other.VF, Other.IsScalable, Other.ISA, Other.Parameters); + } +}; + +/// Holds the VFShape for a specific scalar to vector function mapping. +struct VFInfo { + VFShape Shape; // Classification of the vector function. + StringRef ScalarName; // Scalar Function Name. + StringRef VectorName; // Vector Function Name associated to this VFInfo. + + // Comparison operator. + bool operator==(const VFInfo &Other) const { + return std::tie(Shape, ScalarName, VectorName) == + std::tie(Shape, Other.ScalarName, Other.VectorName); + } +}; + +namespace VFABI { +/// Function to contruct a VFInfo out of a mangled names in the +/// following format: +/// +/// <VFABI_name>{(<redirection>)} +/// +/// where <VFABI_name> is the name of the vector function, mangled according +/// to the rules described in the Vector Function ABI of the target vector +/// extentsion (or <isa> from now on). The <VFABI_name> is in the following +/// format: +/// +/// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)] +/// +/// This methods support demangling rules for the following <isa>: +/// +/// * AArch64: https://developer.arm.com/docs/101129/latest +/// +/// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and +/// https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt +/// +/// +/// +/// \param MangledName -> input string in the format +/// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]. +Optional<VFInfo> tryDemangleForVFABI(StringRef MangledName); + +/// Retrieve the `VFParamKind` from a string token. +VFParamKind getVFParamKindFromString(const StringRef Token); +} // end namespace VFABI + template <typename T> class ArrayRef; class DemandedBits; class GetElementPtrInst; template <typename InstTy> class InterleaveGroup; class Loop; class ScalarEvolution; +class TargetLibraryInfo; class TargetTransformInfo; class Type; class Value; @@ -270,13 +381,12 @@ APInt possiblyDemandedEltsInMask(Value *Mask); /// the interleaved store group doesn't allow gaps. template <typename InstTy> class InterleaveGroup { public: - InterleaveGroup(uint32_t Factor, bool Reverse, uint32_t Align) - : Factor(Factor), Reverse(Reverse), Align(Align), InsertPos(nullptr) {} - - InterleaveGroup(InstTy *Instr, int32_t Stride, uint32_t Align) - : Align(Align), InsertPos(Instr) { - assert(Align && "The alignment should be non-zero"); + InterleaveGroup(uint32_t Factor, bool Reverse, Align Alignment) + : Factor(Factor), Reverse(Reverse), Alignment(Alignment), + InsertPos(nullptr) {} + InterleaveGroup(InstTy *Instr, int32_t Stride, Align Alignment) + : Alignment(Alignment), InsertPos(Instr) { Factor = std::abs(Stride); assert(Factor > 1 && "Invalid interleave factor"); @@ -286,7 +396,7 @@ public: bool isReverse() const { return Reverse; } uint32_t getFactor() const { return Factor; } - uint32_t getAlignment() const { return Align; } + uint32_t getAlignment() const { return Alignment.value(); } uint32_t getNumMembers() const { return Members.size(); } /// Try to insert a new member \p Instr with index \p Index and @@ -294,9 +404,7 @@ public: /// negative if it is the new leader. /// /// \returns false if the instruction doesn't belong to the group. - bool insertMember(InstTy *Instr, int32_t Index, uint32_t NewAlign) { - assert(NewAlign && "The new member's alignment should be non-zero"); - + bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign) { // Make sure the key fits in an int32_t. Optional<int32_t> MaybeKey = checkedAdd(Index, SmallestKey); if (!MaybeKey) @@ -328,7 +436,7 @@ public: } // It's always safe to select the minimum alignment. - Align = std::min(Align, NewAlign); + Alignment = std::min(Alignment, NewAlign); Members[Key] = Instr; return true; } @@ -387,7 +495,7 @@ public: private: uint32_t Factor; // Interleave Factor. bool Reverse; - uint32_t Align; + Align Alignment; DenseMap<int32_t, InstTy *> Members; int32_t SmallestKey = 0; int32_t LargestKey = 0; @@ -504,8 +612,8 @@ private: struct StrideDescriptor { StrideDescriptor() = default; StrideDescriptor(int64_t Stride, const SCEV *Scev, uint64_t Size, - unsigned Align) - : Stride(Stride), Scev(Scev), Size(Size), Align(Align) {} + Align Alignment) + : Stride(Stride), Scev(Scev), Size(Size), Alignment(Alignment) {} // The access's stride. It is negative for a reverse access. int64_t Stride = 0; @@ -517,7 +625,7 @@ private: uint64_t Size = 0; // The alignment of this access. - unsigned Align = 0; + Align Alignment; }; /// A type for holding instructions and their stride descriptors. @@ -528,11 +636,11 @@ private: /// /// \returns the newly created interleave group. InterleaveGroup<Instruction> * - createInterleaveGroup(Instruction *Instr, int Stride, unsigned Align) { + createInterleaveGroup(Instruction *Instr, int Stride, Align Alignment) { assert(!InterleaveGroupMap.count(Instr) && "Already in an interleaved access group"); InterleaveGroupMap[Instr] = - new InterleaveGroup<Instruction>(Instr, Stride, Align); + new InterleaveGroup<Instruction>(Instr, Stride, Alignment); InterleaveGroups.insert(InterleaveGroupMap[Instr]); return InterleaveGroupMap[Instr]; } diff --git a/include/llvm/BinaryFormat/Dwarf.def b/include/llvm/BinaryFormat/Dwarf.def index b0f78d0fd61f..34a7410f7474 100644 --- a/include/llvm/BinaryFormat/Dwarf.def +++ b/include/llvm/BinaryFormat/Dwarf.def @@ -17,7 +17,7 @@ defined HANDLE_DW_VIRTUALITY || defined HANDLE_DW_DEFAULTED || \ defined HANDLE_DW_CC || defined HANDLE_DW_LNS || defined HANDLE_DW_LNE || \ defined HANDLE_DW_LNCT || defined HANDLE_DW_MACRO || \ - defined HANDLE_DW_RLE || \ + defined HANDLE_DW_RLE || defined HANDLE_DW_LLE || \ (defined HANDLE_DW_CFA && defined HANDLE_DW_CFA_PRED) || \ defined HANDLE_DW_APPLE_PROPERTY || defined HANDLE_DW_UT || \ defined HANDLE_DWARF_SECTION || defined HANDLE_DW_IDX || \ @@ -26,7 +26,17 @@ #endif #ifndef HANDLE_DW_TAG -#define HANDLE_DW_TAG(ID, NAME, VERSION, VENDOR) +#define HANDLE_DW_TAG(ID, NAME, VERSION, VENDOR, KIND) +#endif + +// Note that DW_KIND is not a DWARF concept, but rather a way for us to +// generate a list of tags that belong together. +#ifndef DW_KIND_NONE +#define DW_KIND_NONE 0 +#endif + +#ifndef DW_KIND_TYPE +#define DW_KIND_TYPE 1 #endif #ifndef HANDLE_DW_AT @@ -81,6 +91,10 @@ #define HANDLE_DW_RLE(ID, NAME) #endif +#ifndef HANDLE_DW_LLE +#define HANDLE_DW_LLE(ID, NAME) +#endif + #ifndef HANDLE_DW_CFA #define HANDLE_DW_CFA(ID, NAME) #endif @@ -109,94 +123,94 @@ #define HANDLE_DW_END(ID, NAME) #endif -HANDLE_DW_TAG(0x0000, null, 2, DWARF) -HANDLE_DW_TAG(0x0001, array_type, 2, DWARF) -HANDLE_DW_TAG(0x0002, class_type, 2, DWARF) -HANDLE_DW_TAG(0x0003, entry_point, 2, DWARF) -HANDLE_DW_TAG(0x0004, enumeration_type, 2, DWARF) -HANDLE_DW_TAG(0x0005, formal_parameter, 2, DWARF) -HANDLE_DW_TAG(0x0008, imported_declaration, 2, DWARF) -HANDLE_DW_TAG(0x000a, label, 2, DWARF) -HANDLE_DW_TAG(0x000b, lexical_block, 2, DWARF) -HANDLE_DW_TAG(0x000d, member, 2, DWARF) -HANDLE_DW_TAG(0x000f, pointer_type, 2, DWARF) -HANDLE_DW_TAG(0x0010, reference_type, 2, DWARF) -HANDLE_DW_TAG(0x0011, compile_unit, 2, DWARF) -HANDLE_DW_TAG(0x0012, string_type, 2, DWARF) -HANDLE_DW_TAG(0x0013, structure_type, 2, DWARF) -HANDLE_DW_TAG(0x0015, subroutine_type, 2, DWARF) -HANDLE_DW_TAG(0x0016, typedef, 2, DWARF) -HANDLE_DW_TAG(0x0017, union_type, 2, DWARF) -HANDLE_DW_TAG(0x0018, unspecified_parameters, 2, DWARF) -HANDLE_DW_TAG(0x0019, variant, 2, DWARF) -HANDLE_DW_TAG(0x001a, common_block, 2, DWARF) -HANDLE_DW_TAG(0x001b, common_inclusion, 2, DWARF) -HANDLE_DW_TAG(0x001c, inheritance, 2, DWARF) -HANDLE_DW_TAG(0x001d, inlined_subroutine, 2, DWARF) -HANDLE_DW_TAG(0x001e, module, 2, DWARF) -HANDLE_DW_TAG(0x001f, ptr_to_member_type, 2, DWARF) -HANDLE_DW_TAG(0x0020, set_type, 2, DWARF) -HANDLE_DW_TAG(0x0021, subrange_type, 2, DWARF) -HANDLE_DW_TAG(0x0022, with_stmt, 2, DWARF) -HANDLE_DW_TAG(0x0023, access_declaration, 2, DWARF) -HANDLE_DW_TAG(0x0024, base_type, 2, DWARF) -HANDLE_DW_TAG(0x0025, catch_block, 2, DWARF) -HANDLE_DW_TAG(0x0026, const_type, 2, DWARF) -HANDLE_DW_TAG(0x0027, constant, 2, DWARF) -HANDLE_DW_TAG(0x0028, enumerator, 2, DWARF) -HANDLE_DW_TAG(0x0029, file_type, 2, DWARF) -HANDLE_DW_TAG(0x002a, friend, 2, DWARF) -HANDLE_DW_TAG(0x002b, namelist, 2, DWARF) -HANDLE_DW_TAG(0x002c, namelist_item, 2, DWARF) -HANDLE_DW_TAG(0x002d, packed_type, 2, DWARF) -HANDLE_DW_TAG(0x002e, subprogram, 2, DWARF) -HANDLE_DW_TAG(0x002f, template_type_parameter, 2, DWARF) -HANDLE_DW_TAG(0x0030, template_value_parameter, 2, DWARF) -HANDLE_DW_TAG(0x0031, thrown_type, 2, DWARF) -HANDLE_DW_TAG(0x0032, try_block, 2, DWARF) -HANDLE_DW_TAG(0x0033, variant_part, 2, DWARF) -HANDLE_DW_TAG(0x0034, variable, 2, DWARF) -HANDLE_DW_TAG(0x0035, volatile_type, 2, DWARF) +HANDLE_DW_TAG(0x0000, null, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0001, array_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0002, class_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0003, entry_point, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0004, enumeration_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0005, formal_parameter, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0008, imported_declaration, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x000a, label, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x000b, lexical_block, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x000d, member, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x000f, pointer_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0010, reference_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0011, compile_unit, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0012, string_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0013, structure_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0015, subroutine_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0016, typedef, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0017, union_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0018, unspecified_parameters, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0019, variant, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001a, common_block, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001b, common_inclusion, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001c, inheritance, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001d, inlined_subroutine, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001e, module, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x001f, ptr_to_member_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0020, set_type, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0021, subrange_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0022, with_stmt, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0023, access_declaration, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0024, base_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0025, catch_block, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0026, const_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0027, constant, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0028, enumerator, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0029, file_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x002a, friend, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x002b, namelist, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x002c, namelist_item, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x002d, packed_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x002e, subprogram, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x002f, template_type_parameter, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0030, template_value_parameter, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0031, thrown_type, 2, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0032, try_block, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0033, variant_part, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0034, variable, 2, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0035, volatile_type, 2, DWARF, DW_KIND_TYPE) // New in DWARF v3: -HANDLE_DW_TAG(0x0036, dwarf_procedure, 3, DWARF) -HANDLE_DW_TAG(0x0037, restrict_type, 3, DWARF) -HANDLE_DW_TAG(0x0038, interface_type, 3, DWARF) -HANDLE_DW_TAG(0x0039, namespace, 3, DWARF) -HANDLE_DW_TAG(0x003a, imported_module, 3, DWARF) -HANDLE_DW_TAG(0x003b, unspecified_type, 3, DWARF) -HANDLE_DW_TAG(0x003c, partial_unit, 3, DWARF) -HANDLE_DW_TAG(0x003d, imported_unit, 3, DWARF) -HANDLE_DW_TAG(0x003f, condition, 3, DWARF) -HANDLE_DW_TAG(0x0040, shared_type, 3, DWARF) +HANDLE_DW_TAG(0x0036, dwarf_procedure, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0037, restrict_type, 3, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0038, interface_type, 3, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0039, namespace, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x003a, imported_module, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x003b, unspecified_type, 3, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x003c, partial_unit, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x003d, imported_unit, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x003f, condition, 3, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0040, shared_type, 3, DWARF, DW_KIND_TYPE) // New in DWARF v4: -HANDLE_DW_TAG(0x0041, type_unit, 4, DWARF) -HANDLE_DW_TAG(0x0042, rvalue_reference_type, 4, DWARF) -HANDLE_DW_TAG(0x0043, template_alias, 4, DWARF) +HANDLE_DW_TAG(0x0041, type_unit, 4, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0042, rvalue_reference_type, 4, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0043, template_alias, 4, DWARF, DW_KIND_NONE) // New in DWARF v5: -HANDLE_DW_TAG(0x0044, coarray_type, 5, DWARF) -HANDLE_DW_TAG(0x0045, generic_subrange, 5, DWARF) -HANDLE_DW_TAG(0x0046, dynamic_type, 5, DWARF) -HANDLE_DW_TAG(0x0047, atomic_type, 5, DWARF) -HANDLE_DW_TAG(0x0048, call_site, 5, DWARF) -HANDLE_DW_TAG(0x0049, call_site_parameter, 5, DWARF) -HANDLE_DW_TAG(0x004a, skeleton_unit, 5, DWARF) -HANDLE_DW_TAG(0x004b, immutable_type, 5, DWARF) +HANDLE_DW_TAG(0x0044, coarray_type, 5, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0045, generic_subrange, 5, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0046, dynamic_type, 5, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0047, atomic_type, 5, DWARF, DW_KIND_TYPE) +HANDLE_DW_TAG(0x0048, call_site, 5, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x0049, call_site_parameter, 5, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x004a, skeleton_unit, 5, DWARF, DW_KIND_NONE) +HANDLE_DW_TAG(0x004b, immutable_type, 5, DWARF, DW_KIND_TYPE) // Vendor extensions: -HANDLE_DW_TAG(0x4081, MIPS_loop, 0, MIPS) -HANDLE_DW_TAG(0x4101, format_label, 0, GNU) -HANDLE_DW_TAG(0x4102, function_template, 0, GNU) -HANDLE_DW_TAG(0x4103, class_template, 0, GNU) -HANDLE_DW_TAG(0x4106, GNU_template_template_param, 0, GNU) -HANDLE_DW_TAG(0x4107, GNU_template_parameter_pack, 0, GNU) -HANDLE_DW_TAG(0x4108, GNU_formal_parameter_pack, 0, GNU) -HANDLE_DW_TAG(0x4109, GNU_call_site, 0, GNU) -HANDLE_DW_TAG(0x410a, GNU_call_site_parameter, 0, GNU) -HANDLE_DW_TAG(0x4200, APPLE_property, 0, APPLE) -HANDLE_DW_TAG(0xb000, BORLAND_property, 0, BORLAND) -HANDLE_DW_TAG(0xb001, BORLAND_Delphi_string, 0, BORLAND) -HANDLE_DW_TAG(0xb002, BORLAND_Delphi_dynamic_array, 0, BORLAND) -HANDLE_DW_TAG(0xb003, BORLAND_Delphi_set, 0, BORLAND) -HANDLE_DW_TAG(0xb004, BORLAND_Delphi_variant, 0, BORLAND) +HANDLE_DW_TAG(0x4081, MIPS_loop, 0, MIPS, DW_KIND_NONE) +HANDLE_DW_TAG(0x4101, format_label, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4102, function_template, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4103, class_template, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4106, GNU_template_template_param, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4107, GNU_template_parameter_pack, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4108, GNU_formal_parameter_pack, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4109, GNU_call_site, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x410a, GNU_call_site_parameter, 0, GNU, DW_KIND_NONE) +HANDLE_DW_TAG(0x4200, APPLE_property, 0, APPLE, DW_KIND_NONE) +HANDLE_DW_TAG(0xb000, BORLAND_property, 0, BORLAND, DW_KIND_NONE) +HANDLE_DW_TAG(0xb001, BORLAND_Delphi_string, 0, BORLAND, DW_KIND_TYPE) +HANDLE_DW_TAG(0xb002, BORLAND_Delphi_dynamic_array, 0, BORLAND, DW_KIND_TYPE) +HANDLE_DW_TAG(0xb003, BORLAND_Delphi_set, 0, BORLAND, DW_KIND_TYPE) +HANDLE_DW_TAG(0xb004, BORLAND_Delphi_variant, 0, BORLAND, DW_KIND_TYPE) // Attributes. HANDLE_DW_AT(0x01, sibling, 2, DWARF) @@ -815,6 +829,17 @@ HANDLE_DW_RLE(0x05, base_address) HANDLE_DW_RLE(0x06, start_end) HANDLE_DW_RLE(0x07, start_length) +// DWARF v5 Loc List Entry encoding values. +HANDLE_DW_LLE(0x00, end_of_list) +HANDLE_DW_LLE(0x01, base_addressx) +HANDLE_DW_LLE(0x02, startx_endx) +HANDLE_DW_LLE(0x03, startx_length) +HANDLE_DW_LLE(0x04, offset_pair) +HANDLE_DW_LLE(0x05, default_location) +HANDLE_DW_LLE(0x06, base_address) +HANDLE_DW_LLE(0x07, start_end) +HANDLE_DW_LLE(0x08, start_length) + // Call frame instruction encodings. HANDLE_DW_CFA(0x00, nop) HANDLE_DW_CFA(0x40, advance_loc) @@ -929,6 +954,7 @@ HANDLE_DW_IDX(0x05, type_hash) #undef HANDLE_DW_LNCT #undef HANDLE_DW_MACRO #undef HANDLE_DW_RLE +#undef HANDLE_DW_LLE #undef HANDLE_DW_CFA #undef HANDLE_DW_CFA_PRED #undef HANDLE_DW_APPLE_PROPERTY diff --git a/include/llvm/BinaryFormat/Dwarf.h b/include/llvm/BinaryFormat/Dwarf.h index 76d9c365c0a8..1c6aee48661c 100644 --- a/include/llvm/BinaryFormat/Dwarf.h +++ b/include/llvm/BinaryFormat/Dwarf.h @@ -46,6 +46,11 @@ enum LLVMConstants : uint32_t { DW_VIRTUALITY_invalid = ~0U, // Virtuality for invalid results. DW_MACINFO_invalid = ~0U, // Macinfo type for invalid results. + // Special values for an initial length field. + DW_LENGTH_lo_reserved = 0xfffffff0, // Lower bound of the reserved range. + DW_LENGTH_DWARF64 = 0xffffffff, // Indicator of 64-bit DWARF format. + DW_LENGTH_hi_reserved = 0xffffffff, // Upper bound of the reserved range. + // Other constants. DWARF_VERSION = 4, // Default dwarf version we output. DW_PUBTYPES_VERSION = 2, // Section version number for .debug_pubtypes. @@ -75,7 +80,7 @@ const uint64_t DW64_CIE_ID = UINT64_MAX; const uint32_t DW_INVALID_OFFSET = UINT32_MAX; enum Tag : uint16_t { -#define HANDLE_DW_TAG(ID, NAME, VERSION, VENDOR) DW_TAG_##NAME = ID, +#define HANDLE_DW_TAG(ID, NAME, VERSION, VENDOR, KIND) DW_TAG_##NAME = ID, #include "llvm/BinaryFormat/Dwarf.def" DW_TAG_lo_user = 0x4080, DW_TAG_hi_user = 0xffff, @@ -84,29 +89,12 @@ enum Tag : uint16_t { inline bool isType(Tag T) { switch (T) { - case DW_TAG_array_type: - case DW_TAG_class_type: - case DW_TAG_interface_type: - case DW_TAG_enumeration_type: - case DW_TAG_pointer_type: - case DW_TAG_reference_type: - case DW_TAG_rvalue_reference_type: - case DW_TAG_string_type: - case DW_TAG_structure_type: - case DW_TAG_subroutine_type: - case DW_TAG_union_type: - case DW_TAG_ptr_to_member_type: - case DW_TAG_set_type: - case DW_TAG_subrange_type: - case DW_TAG_base_type: - case DW_TAG_const_type: - case DW_TAG_file_type: - case DW_TAG_packed_type: - case DW_TAG_volatile_type: - case DW_TAG_typedef: - return true; default: return false; +#define HANDLE_DW_TAG(ID, NAME, VERSION, VENDOR, KIND) \ + case DW_TAG_##NAME: \ + return (KIND == DW_KIND_TYPE); +#include "llvm/BinaryFormat/Dwarf.def" } } @@ -129,9 +117,10 @@ enum LocationAtom { #include "llvm/BinaryFormat/Dwarf.def" DW_OP_lo_user = 0xe0, DW_OP_hi_user = 0xff, - DW_OP_LLVM_fragment = 0x1000, ///< Only used in LLVM metadata. - DW_OP_LLVM_convert = 0x1001, ///< Only used in LLVM metadata. - DW_OP_LLVM_tag_offset = 0x1002, ///< Only used in LLVM metadata. + DW_OP_LLVM_fragment = 0x1000, ///< Only used in LLVM metadata. + DW_OP_LLVM_convert = 0x1001, ///< Only used in LLVM metadata. + DW_OP_LLVM_tag_offset = 0x1002, ///< Only used in LLVM metadata. + DW_OP_LLVM_entry_value = 0x1003, ///< Only used in LLVM metadata. }; enum TypeKind : uint8_t { @@ -192,6 +181,59 @@ enum SourceLanguage { DW_LANG_hi_user = 0xffff }; +inline bool isCPlusPlus(SourceLanguage S) { + // Deliberately enumerate all the language options so we get a warning when + // new language options are added (-Wswitch) that'll hopefully help keep this + // switch up-to-date when new C++ versions are added. + switch (S) { + case DW_LANG_C_plus_plus: + case DW_LANG_C_plus_plus_03: + case DW_LANG_C_plus_plus_11: + case DW_LANG_C_plus_plus_14: + return true; + case DW_LANG_C89: + case DW_LANG_C: + case DW_LANG_Ada83: + case DW_LANG_Cobol74: + case DW_LANG_Cobol85: + case DW_LANG_Fortran77: + case DW_LANG_Fortran90: + case DW_LANG_Pascal83: + case DW_LANG_Modula2: + case DW_LANG_Java: + case DW_LANG_C99: + case DW_LANG_Ada95: + case DW_LANG_Fortran95: + case DW_LANG_PLI: + case DW_LANG_ObjC: + case DW_LANG_ObjC_plus_plus: + case DW_LANG_UPC: + case DW_LANG_D: + case DW_LANG_Python: + case DW_LANG_OpenCL: + case DW_LANG_Go: + case DW_LANG_Modula3: + case DW_LANG_Haskell: + case DW_LANG_OCaml: + case DW_LANG_Rust: + case DW_LANG_C11: + case DW_LANG_Swift: + case DW_LANG_Julia: + case DW_LANG_Dylan: + case DW_LANG_Fortran03: + case DW_LANG_Fortran08: + case DW_LANG_RenderScript: + case DW_LANG_BLISS: + case DW_LANG_Mips_Assembler: + case DW_LANG_GOOGLE_RenderScript: + case DW_LANG_BORLAND_Delphi: + case DW_LANG_lo_user: + case DW_LANG_hi_user: + return false; + } + llvm_unreachable("Invalid source language"); +} + enum CaseSensitivity { // Identifier case codes DW_ID_case_sensitive = 0x00, @@ -267,11 +309,17 @@ enum MacroEntryType { }; /// DWARF v5 range list entry encoding values. -enum RangeListEntries { +enum RnglistEntries { #define HANDLE_DW_RLE(ID, NAME) DW_RLE_##NAME = ID, #include "llvm/BinaryFormat/Dwarf.def" }; +/// DWARF v5 loc list entry encoding values. +enum LoclistEntries { +#define HANDLE_DW_LLE(ID, NAME) DW_LLE_##NAME = ID, +#include "llvm/BinaryFormat/Dwarf.def" +}; + /// Call frame instruction encodings. enum CallFrameInfo { #define HANDLE_DW_CFA(ID, NAME) DW_CFA_##NAME = ID, @@ -307,19 +355,6 @@ enum Constants { DW_EH_PE_indirect = 0x80 }; -/// Constants for location lists in DWARF v5. -enum LocationListEntry : unsigned char { - DW_LLE_end_of_list = 0x00, - DW_LLE_base_addressx = 0x01, - DW_LLE_startx_endx = 0x02, - DW_LLE_startx_length = 0x03, - DW_LLE_offset_pair = 0x04, - DW_LLE_default_location = 0x05, - DW_LLE_base_address = 0x06, - DW_LLE_start_end = 0x07, - DW_LLE_start_length = 0x08 -}; - /// Constants for the DW_APPLE_PROPERTY_attributes attribute. /// Keep this list in sync with clang's DeclSpec.h ObjCPropertyAttributeKind! enum ApplePropertyAttributes { @@ -434,6 +469,7 @@ StringRef LNStandardString(unsigned Standard); StringRef LNExtendedString(unsigned Encoding); StringRef MacinfoString(unsigned Encoding); StringRef RangeListEncodingString(unsigned Encoding); +StringRef LocListEncodingString(unsigned Encoding); StringRef CallFrameString(unsigned Encoding, Triple::ArchType Arch); StringRef ApplePropertyString(unsigned); StringRef UnitTypeString(unsigned); @@ -525,6 +561,17 @@ struct FormParams { explicit operator bool() const { return Version && AddrSize; } }; +/// Get the byte size of the unit length field depending on the DWARF format. +inline uint8_t getUnitLengthFieldByteSize(DwarfFormat Format) { + switch (Format) { + case DwarfFormat::DWARF32: + return 4; + case DwarfFormat::DWARF64: + return 12; + } + llvm_unreachable("Invalid Format value"); +} + /// Get the fixed byte size for a given form. /// /// If the form has a fixed byte size, then an Optional with a value will be diff --git a/include/llvm/BinaryFormat/ELF.h b/include/llvm/BinaryFormat/ELF.h index 2bd711137845..46edfb6260be 100644 --- a/include/llvm/BinaryFormat/ELF.h +++ b/include/llvm/BinaryFormat/ELF.h @@ -1356,6 +1356,72 @@ enum : unsigned { NT_GNU_BUILD_ATTRIBUTE_FUNC = 0x101, }; +// Core note types +enum : unsigned { + NT_PRSTATUS = 1, + NT_FPREGSET = 2, + NT_PRPSINFO = 3, + NT_TASKSTRUCT = 4, + NT_AUXV = 6, + NT_PSTATUS = 10, + NT_FPREGS = 12, + NT_PSINFO = 13, + NT_LWPSTATUS = 16, + NT_LWPSINFO = 17, + NT_WIN32PSTATUS = 18, + + NT_PPC_VMX = 0x100, + NT_PPC_VSX = 0x102, + NT_PPC_TAR = 0x103, + NT_PPC_PPR = 0x104, + NT_PPC_DSCR = 0x105, + NT_PPC_EBB = 0x106, + NT_PPC_PMU = 0x107, + NT_PPC_TM_CGPR = 0x108, + NT_PPC_TM_CFPR = 0x109, + NT_PPC_TM_CVMX = 0x10a, + NT_PPC_TM_CVSX = 0x10b, + NT_PPC_TM_SPR = 0x10c, + NT_PPC_TM_CTAR = 0x10d, + NT_PPC_TM_CPPR = 0x10e, + NT_PPC_TM_CDSCR = 0x10f, + + NT_386_TLS = 0x200, + NT_386_IOPERM = 0x201, + NT_X86_XSTATE = 0x202, + + NT_S390_HIGH_GPRS = 0x300, + NT_S390_TIMER = 0x301, + NT_S390_TODCMP = 0x302, + NT_S390_TODPREG = 0x303, + NT_S390_CTRS = 0x304, + NT_S390_PREFIX = 0x305, + NT_S390_LAST_BREAK = 0x306, + NT_S390_SYSTEM_CALL = 0x307, + NT_S390_TDB = 0x308, + NT_S390_VXRS_LOW = 0x309, + NT_S390_VXRS_HIGH = 0x30a, + NT_S390_GS_CB = 0x30b, + NT_S390_GS_BC = 0x30c, + + NT_ARM_VFP = 0x400, + NT_ARM_TLS = 0x401, + NT_ARM_HW_BREAK = 0x402, + NT_ARM_HW_WATCH = 0x403, + NT_ARM_SVE = 0x405, + NT_ARM_PAC_MASK = 0x406, + + NT_FILE = 0x46494c45, + NT_PRXFPREG = 0x46e62b7f, + NT_SIGINFO = 0x53494749, +}; + +// LLVM-specific notes. +enum { + NT_LLVM_HWASAN_GLOBALS = 3, +}; + +// GNU note types enum { NT_GNU_ABI_TAG = 1, NT_GNU_HWCAP = 2, diff --git a/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/include/llvm/BinaryFormat/ELFRelocs/AArch64.def index 4afcd7d1f093..c8364133e31f 100644 --- a/include/llvm/BinaryFormat/ELFRelocs/AArch64.def +++ b/include/llvm/BinaryFormat/ELFRelocs/AArch64.def @@ -124,8 +124,11 @@ ELF_RELOC(R_AARCH64_COPY, 0x400) ELF_RELOC(R_AARCH64_GLOB_DAT, 0x401) ELF_RELOC(R_AARCH64_JUMP_SLOT, 0x402) ELF_RELOC(R_AARCH64_RELATIVE, 0x403) -ELF_RELOC(R_AARCH64_TLS_DTPREL64, 0x404) -ELF_RELOC(R_AARCH64_TLS_DTPMOD64, 0x405) +// 0x404 and 0x405 are now R_AARCH64_TLS_IMPDEF1 and R_AARCH64_TLS_IMPDEF2 +// We follow GNU and define TLS_IMPDEF1 as TLS_DTPMOD64 and TLS_IMPDEF2 as +// TLS_DTPREL64 +ELF_RELOC(R_AARCH64_TLS_DTPMOD64, 0x404) +ELF_RELOC(R_AARCH64_TLS_DTPREL64, 0x405) ELF_RELOC(R_AARCH64_TLS_TPREL64, 0x406) ELF_RELOC(R_AARCH64_TLSDESC, 0x407) ELF_RELOC(R_AARCH64_IRELATIVE, 0x408) diff --git a/include/llvm/BinaryFormat/MachO.h b/include/llvm/BinaryFormat/MachO.h index a01393a3b303..fb50e549cb9d 100644 --- a/include/llvm/BinaryFormat/MachO.h +++ b/include/llvm/BinaryFormat/MachO.h @@ -581,6 +581,11 @@ struct section_64 { uint32_t reserved3; }; +inline bool isVirtualSection(uint8_t type) { + return (type == MachO::S_ZEROFILL || type == MachO::S_GB_ZEROFILL || + type == MachO::S_THREAD_LOCAL_ZEROFILL); +} + struct fvmlib { uint32_t name; uint32_t minor_version; diff --git a/include/llvm/BinaryFormat/Magic.h b/include/llvm/BinaryFormat/Magic.h index cd9833ec4d22..64c687262f4a 100644 --- a/include/llvm/BinaryFormat/Magic.h +++ b/include/llvm/BinaryFormat/Magic.h @@ -49,6 +49,7 @@ struct file_magic { xcoff_object_64, ///< 64-bit XCOFF object file wasm_object, ///< WebAssembly Object file pdb, ///< Windows PDB debug info file + tapi_file, ///< Text-based Dynamic Library Stub file }; bool is_object() const { return V != unknown; } diff --git a/include/llvm/BinaryFormat/Minidump.h b/include/llvm/BinaryFormat/Minidump.h index 65c17d1eb00c..89cd779951cf 100644 --- a/include/llvm/BinaryFormat/Minidump.h +++ b/include/llvm/BinaryFormat/Minidump.h @@ -18,12 +18,15 @@ #ifndef LLVM_BINARYFORMAT_MINIDUMP_H #define LLVM_BINARYFORMAT_MINIDUMP_H +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/Endian.h" namespace llvm { namespace minidump { +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); + /// The minidump header is the first part of a minidump file. It identifies the /// file as a minidump file, and gives the location of the stream directory. struct Header { @@ -67,6 +70,50 @@ struct MemoryDescriptor { }; static_assert(sizeof(MemoryDescriptor) == 16, ""); +struct MemoryInfoListHeader { + support::ulittle32_t SizeOfHeader; + support::ulittle32_t SizeOfEntry; + support::ulittle64_t NumberOfEntries; + + MemoryInfoListHeader() = default; + MemoryInfoListHeader(uint32_t SizeOfHeader, uint32_t SizeOfEntry, + uint64_t NumberOfEntries) + : SizeOfHeader(SizeOfHeader), SizeOfEntry(SizeOfEntry), + NumberOfEntries(NumberOfEntries) {} +}; +static_assert(sizeof(MemoryInfoListHeader) == 16, ""); + +enum class MemoryProtection : uint32_t { +#define HANDLE_MDMP_PROTECT(CODE, NAME, NATIVENAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/0xffffffffu), +}; + +enum class MemoryState : uint32_t { +#define HANDLE_MDMP_MEMSTATE(CODE, NAME, NATIVENAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/0xffffffffu), +}; + +enum class MemoryType : uint32_t { +#define HANDLE_MDMP_MEMTYPE(CODE, NAME, NATIVENAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/0xffffffffu), +}; + +struct MemoryInfo { + support::ulittle64_t BaseAddress; + support::ulittle64_t AllocationBase; + support::little_t<MemoryProtection> AllocationProtect; + support::ulittle32_t Reserved0; + support::ulittle64_t RegionSize; + support::little_t<MemoryState> State; + support::little_t<MemoryProtection> Protect; + support::little_t<MemoryType> Type; + support::ulittle32_t Reserved1; +}; +static_assert(sizeof(MemoryInfo) == 48, ""); + /// Specifies the location and type of a single stream in the minidump file. The /// minidump stream directory is an array of entries of this type, with its size /// given by Header.NumberOfStreams. @@ -180,6 +227,27 @@ struct Thread { }; static_assert(sizeof(Thread) == 48, ""); +struct Exception { + static constexpr size_t MaxParameters = 15; + + support::ulittle32_t ExceptionCode; + support::ulittle32_t ExceptionFlags; + support::ulittle64_t ExceptionRecord; + support::ulittle64_t ExceptionAddress; + support::ulittle32_t NumberParameters; + support::ulittle32_t UnusedAlignment; + support::ulittle64_t ExceptionInformation[MaxParameters]; +}; +static_assert(sizeof(Exception) == 152, ""); + +struct ExceptionStream { + support::ulittle32_t ThreadId; + support::ulittle32_t UnusedAlignment; + Exception ExceptionRecord; + LocationDescriptor ThreadContext; +}; +static_assert(sizeof(ExceptionStream) == 168, ""); + } // namespace minidump template <> struct DenseMapInfo<minidump::StreamType> { diff --git a/include/llvm/BinaryFormat/MinidumpConstants.def b/include/llvm/BinaryFormat/MinidumpConstants.def index d4f13dd99217..aeef399af7a4 100644 --- a/include/llvm/BinaryFormat/MinidumpConstants.def +++ b/include/llvm/BinaryFormat/MinidumpConstants.def @@ -6,8 +6,9 @@ // //===----------------------------------------------------------------------===// -#if !(defined HANDLE_MDMP_STREAM_TYPE || defined HANDLE_MDMP_ARCH || \ - defined HANDLE_MDMP_PLATFORM) +#if !(defined(HANDLE_MDMP_STREAM_TYPE) || defined(HANDLE_MDMP_ARCH) || \ + defined(HANDLE_MDMP_PLATFORM) || defined(HANDLE_MDMP_PROTECT) || \ + defined(HANDLE_MDMP_MEMSTATE) || defined(HANDLE_MDMP_MEMTYPE)) #error "Missing HANDLE_MDMP definition" #endif @@ -23,6 +24,18 @@ #define HANDLE_MDMP_PLATFORM(CODE, NAME) #endif +#ifndef HANDLE_MDMP_PROTECT +#define HANDLE_MDMP_PROTECT(CODE, NAME, NATIVENAME) +#endif + +#ifndef HANDLE_MDMP_MEMSTATE +#define HANDLE_MDMP_MEMSTATE(CODE, NAME, NATIVENAME) +#endif + +#ifndef HANDLE_MDMP_MEMTYPE +#define HANDLE_MDMP_MEMTYPE(CODE, NAME, NATIVENAME) +#endif + HANDLE_MDMP_STREAM_TYPE(0x0003, ThreadList) HANDLE_MDMP_STREAM_TYPE(0x0004, ModuleList) HANDLE_MDMP_STREAM_TYPE(0x0005, MemoryList) @@ -102,6 +115,30 @@ HANDLE_MDMP_PLATFORM(0x8203, Android) // Android HANDLE_MDMP_PLATFORM(0x8204, PS3) // PS3 HANDLE_MDMP_PLATFORM(0x8205, NaCl) // Native Client (NaCl) +HANDLE_MDMP_PROTECT(0x01, NoAccess, PAGE_NO_ACCESS) +HANDLE_MDMP_PROTECT(0x02, ReadOnly, PAGE_READ_ONLY) +HANDLE_MDMP_PROTECT(0x04, ReadWrite, PAGE_READ_WRITE) +HANDLE_MDMP_PROTECT(0x08, WriteCopy, PAGE_WRITE_COPY) +HANDLE_MDMP_PROTECT(0x10, Execute, PAGE_EXECUTE) +HANDLE_MDMP_PROTECT(0x20, ExecuteRead, PAGE_EXECUTE_READ) +HANDLE_MDMP_PROTECT(0x40, ExecuteReadWrite, PAGE_EXECUTE_READ_WRITE) +HANDLE_MDMP_PROTECT(0x80, ExeciteWriteCopy, PAGE_EXECUTE_WRITE_COPY) +HANDLE_MDMP_PROTECT(0x100, Guard, PAGE_GUARD) +HANDLE_MDMP_PROTECT(0x200, NoCache, PAGE_NOCACHE) +HANDLE_MDMP_PROTECT(0x400, WriteCombine, PAGE_WRITECOMBINE) +HANDLE_MDMP_PROTECT(0x40000000, TargetsInvalid, PAGE_TARGETS_INVALID) + +HANDLE_MDMP_MEMSTATE(0x01000, Commit, MEM_COMMIT) +HANDLE_MDMP_MEMSTATE(0x02000, Reserve, MEM_RESERVE) +HANDLE_MDMP_MEMSTATE(0x10000, Free, MEM_FREE) + +HANDLE_MDMP_MEMTYPE(0x0020000, Private, MEM_PRIVATE) +HANDLE_MDMP_MEMTYPE(0x0040000, Mapped, MEM_MAPPED) +HANDLE_MDMP_MEMTYPE(0x1000000, Image, MEM_IMAGE) + #undef HANDLE_MDMP_STREAM_TYPE #undef HANDLE_MDMP_ARCH #undef HANDLE_MDMP_PLATFORM +#undef HANDLE_MDMP_PROTECT +#undef HANDLE_MDMP_MEMSTATE +#undef HANDLE_MDMP_MEMTYPE diff --git a/include/llvm/BinaryFormat/Wasm.h b/include/llvm/BinaryFormat/Wasm.h index 0f22bfe610c6..f550d880f68a 100644 --- a/include/llvm/BinaryFormat/Wasm.h +++ b/include/llvm/BinaryFormat/Wasm.h @@ -16,6 +16,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" namespace llvm { namespace wasm { @@ -251,9 +252,21 @@ enum : unsigned { WASM_OPCODE_F32_CONST = 0x43, WASM_OPCODE_F64_CONST = 0x44, WASM_OPCODE_I32_ADD = 0x6a, +}; + +// Opcodes used in synthetic functions. +enum : unsigned { + WASM_OPCODE_IF = 0x04, + WASM_OPCODE_ELSE = 0x05, + WASM_OPCODE_DROP = 0x1a, WASM_OPCODE_MISC_PREFIX = 0xfc, WASM_OPCODE_MEMORY_INIT = 0x08, WASM_OPCODE_DATA_DROP = 0x09, + WASM_OPCODE_ATOMICS_PREFIX = 0xfe, + WASM_OPCODE_ATOMIC_NOTIFY = 0x00, + WASM_OPCODE_I32_ATOMIC_WAIT = 0x01, + WASM_OPCODE_I32_ATOMIC_STORE = 0x17, + WASM_OPCODE_I32_RMW_CMPXCHG = 0x48, }; enum : unsigned { @@ -318,6 +331,7 @@ const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4; const unsigned WASM_SYMBOL_UNDEFINED = 0x10; const unsigned WASM_SYMBOL_EXPORTED = 0x20; const unsigned WASM_SYMBOL_EXPLICIT_NAME = 0x40; +const unsigned WASM_SYMBOL_NO_STRIP = 0x80; #define WASM_RELOC(name, value) name = value, diff --git a/include/llvm/BinaryFormat/XCOFF.h b/include/llvm/BinaryFormat/XCOFF.h index 7774ab3ed24a..20a0f446272f 100644 --- a/include/llvm/BinaryFormat/XCOFF.h +++ b/include/llvm/BinaryFormat/XCOFF.h @@ -19,12 +19,13 @@ namespace llvm { namespace XCOFF { // Constants used in the XCOFF definition. -enum { SectionNameSize = 8, SymbolNameSize = 8 }; +enum { FileNamePadSize = 6, NameSize = 8, SymbolTableEntrySize = 18 }; + enum ReservedSectionNum { N_DEBUG = -2, N_ABS = -1, N_UNDEF = 0 }; // x_smclas field of x_csect from system header: /usr/include/syms.h /// Storage Mapping Class definitions. -enum StorageMappingClass { +enum StorageMappingClass : uint8_t { // READ ONLY CLASSES XMC_PR = 0, ///< Program Code XMC_RO = 1, ///< Read Only Constant @@ -139,6 +140,117 @@ enum StorageClass : uint8_t { C_TCSYM = 134 // Reserved }; +enum SymbolType { + XTY_ER = 0, ///< External reference. + XTY_SD = 1, ///< Csect definition for initialized storage. + XTY_LD = 2, ///< Label definition. + ///< Defines an entry point to an initialized csect. + XTY_CM = 3 ///< Common csect definition. For uninitialized storage. +}; + +// Relocation types, defined in `/usr/include/reloc.h`. +enum RelocationType : uint8_t { + R_POS = 0x00, ///< Positive relocation. Provides the address of the referenced + ///< symbol. + R_RL = 0x0c, ///< Positive indirect load relocation. Modifiable instruction. + R_RLA = 0x0d, ///< Positive load address relocation. Modifiable instruction. + + R_NEG = 0x01, ///< Negative relocation. Provides the negative of the address + ///< of the referenced symbol. + R_REL = 0x02, ///< Relative to self relocation. Provides a displacement value + ///< between the address of the referenced symbol and the + ///< address being relocated. + + R_TOC = 0x03, ///< Relative to the TOC relocation. Provides a displacement + ///< that is the difference between the address of the + ///< referenced symbol and the TOC anchor csect. + R_TRL = 0x12, ///< TOC relative indirect load relocation. Similar to R_TOC, + ///< but not modifiable instruction. + + R_TRLA = + 0x13, ///< Relative to the TOC or to the thread-local storage base + ///< relocation. Compilers are not permitted to generate this + ///< relocation type. It is the result of a reversible + ///< transformation by the linker of an R_TOC relation that turned a + ///< load instruction into an add-immediate instruction. + + R_GL = 0x05, ///< Global linkage-external TOC address relocation. Provides the + ///< address of the external TOC associated with a defined + ///< external symbol. + R_TCL = 0x06, ///< Local object TOC address relocation. Provides the address + ///< of the local TOC entry of a defined external symbol. + + R_REF = 0x0f, ///< A non-relocating relocation. Used to prevent the binder + ///< from garbage collecting a csect (such as code used for + ///< dynamic initialization of non-local statics) for which + ///< another csect has an implicit dependency. + + R_BA = 0x08, ///< Branch absolute relocation. Provides the address of the + ///< referenced symbol. References a non-modifiable instruction. + R_BR = 0x0a, ///< Branch relative to self relocation. Provides the + ///< displacement that is the difference between the address of + ///< the referenced symbol and the address of the referenced + ///< branch instruction. References a non-modifiable instruction. + R_RBA = 0x18, ///< Branch absolute relocation. Similar to R_BA but + ///< references a modifiable instruction. + R_RBR = 0x1a, ///< Branch relative to self relocation. Similar to the R_BR + ///< relocation type, but references a modifiable instruction. + + R_TLS = 0x20, ///< General-dynamic reference to TLS symbol. + R_TLS_IE = 0x21, ///< Initial-exec reference to TLS symbol. + R_TLS_LD = 0x22, ///< Local-dynamic reference to TLS symbol. + R_TLS_LE = 0x23, ///< Local-exec reference to TLS symbol. + R_TLSM = 0x24, ///< Module reference to TLS. Provides a handle for the module + ///< containing the referenced symbol. + R_TLSML = 0x25, ///< Module reference to the local TLS storage. + + R_TOCU = 0x30, ///< Relative to TOC upper. Specifies the high-order 16 bits of + ///< a large code model TOC-relative relocation. + R_TOCL = 0x31 ///< Relative to TOC lower. Specifies the low-order 16 bits of a + ///< large code model TOC-relative relocation. +}; + +struct FileHeader32 { + uint16_t Magic; + uint16_t NumberOfSections; + int32_t TimeStamp; + uint32_t SymbolTableFileOffset; + int32_t NumberOfSymbolTableEntries; + uint16_t AuxiliaryHeaderSize; + uint16_t Flags; +}; + +struct SectionHeader32 { + char Name[XCOFF::NameSize]; + uint32_t PhysicalAddress; + uint32_t VirtualAddress; + uint32_t Size; + uint32_t FileOffsetToData; + uint32_t FileOffsetToRelocations; + uint32_t FileOffsetToLineNumbers; + uint16_t NumberOfRelocations; + uint16_t NumberOfLineNumbers; + int32_t Flags; +}; + +enum CFileStringType : uint8_t { + XFT_FN = 0, ///< Specifies the source-file name. + XFT_CT = 1, ///< Specifies the compiler time stamp. + XFT_CV = 2, ///< Specifies the compiler version number. + XFT_CD = 128 ///< Specifies compiler-defined information. +}; + +enum CFileLangId : uint8_t { + TB_C = 0, ///< C language. + TB_CPLUSPLUS = 9 ///< C++ language. +}; + +enum CFileCpuId : uint8_t { + TCPU_PPC64 = 2, ///< PowerPC common architecture 64-bit mode. + TCPU_COM = 3, ///< POWER and PowerPC architecture common. + TCPU_970 = 19 ///< PPC970 - PowerPC 64-bit architecture. +}; + } // end namespace XCOFF } // end namespace llvm diff --git a/include/llvm/Bitcode/BitcodeAnalyzer.h b/include/llvm/Bitcode/BitcodeAnalyzer.h index cfdebd6fe6cb..5fb8bb26f255 100644 --- a/include/llvm/Bitcode/BitcodeAnalyzer.h +++ b/include/llvm/Bitcode/BitcodeAnalyzer.h @@ -30,6 +30,7 @@ enum CurStreamTypeType { LLVMIRBitstream, ClangSerializedASTBitstream, ClangSerializedDiagnosticsBitstream, + LLVMBitstreamRemarks }; struct BCDumpOptions { diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h index decd4dd3a965..1a397068caf0 100644 --- a/include/llvm/Bitcode/LLVMBitCodes.h +++ b/include/llvm/Bitcode/LLVMBitCodes.h @@ -391,7 +391,7 @@ enum CastOpcodes { /// have no fixed relation to the LLVM IR enum values. Changing these will /// break compatibility with old files. enum UnaryOpcodes { - UNOP_NEG = 0 + UNOP_FNEG = 0 }; /// BinaryOpcodes - These are values used in the bitcode files to encode which diff --git a/include/llvm/Bitstream/BitCodes.h b/include/llvm/Bitstream/BitCodes.h index adf54ba96396..41a3de3b20ef 100644 --- a/include/llvm/Bitstream/BitCodes.h +++ b/include/llvm/Bitstream/BitCodes.h @@ -168,6 +168,11 @@ class BitCodeAbbrev { SmallVector<BitCodeAbbrevOp, 32> OperandList; public: + BitCodeAbbrev() = default; + + explicit BitCodeAbbrev(std::initializer_list<BitCodeAbbrevOp> OperandList) + : OperandList(OperandList) {} + unsigned getNumOperandInfos() const { return static_cast<unsigned>(OperandList.size()); } diff --git a/include/llvm/Bitstream/BitstreamReader.h b/include/llvm/Bitstream/BitstreamReader.h index ee82e7ec1ba2..b49a969a2d8b 100644 --- a/include/llvm/Bitstream/BitstreamReader.h +++ b/include/llvm/Bitstream/BitstreamReader.h @@ -379,6 +379,7 @@ public: using SimpleBitstreamCursor::ReadVBR; using SimpleBitstreamCursor::ReadVBR64; using SimpleBitstreamCursor::SizeInBytes; + using SimpleBitstreamCursor::skipToEnd; /// Return the number of bits used to encode an abbrev #. unsigned getAbbrevIDWidth() const { return CurCodeSize; } diff --git a/include/llvm/CodeGen/AccelTable.h b/include/llvm/CodeGen/AccelTable.h index 734531a65d50..f8f6b5448f3f 100644 --- a/include/llvm/CodeGen/AccelTable.h +++ b/include/llvm/CodeGen/AccelTable.h @@ -101,8 +101,6 @@ /// /// An Apple Accelerator Table can be serialized by calling emitAppleAccelTable /// function. -/// -/// TODO: Add DWARF v5 emission code. namespace llvm { diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index d110f8b01cb5..a4580da5aec9 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -111,6 +111,10 @@ public: /// of each call to runOnMachineFunction(). MCSymbol *CurrentFnSym = nullptr; + /// The symbol for the current function descriptor on AIX. This is created + /// at the beginning of each call to SetupMachineFunction(). + MCSymbol *CurrentFnDescSym = nullptr; + /// The symbol used to represent the start of the current function for the /// purpose of calculating its size (e.g. using the .size directive). By /// default, this is equal to CurrentFnSym. @@ -304,7 +308,7 @@ public: /// This should be called when a new MachineFunction is being processed from /// runOnMachineFunction. - void SetupMachineFunction(MachineFunction &MF); + virtual void SetupMachineFunction(MachineFunction &MF); /// This method emits the body and trailer for a function. void EmitFunctionBody(); @@ -342,12 +346,11 @@ public: /// so, emit it and return true, otherwise do nothing and return false. bool EmitSpecialLLVMGlobal(const GlobalVariable *GV); - /// Emit an alignment directive to the specified power of two boundary. For - /// example, if you pass in 3 here, you will get an 8 byte alignment. If a + /// Emit an alignment directive to the specified power of two boundary. If a /// global value is specified, and if that global has an explicit alignment /// requested, it will override the alignment request if required for /// correctness. - void EmitAlignment(unsigned NumBits, const GlobalObject *GV = nullptr) const; + void EmitAlignment(Align Alignment, const GlobalObject *GV = nullptr) const; /// Lower the specified LLVM Constant to an MCExpr. virtual const MCExpr *lowerConstant(const Constant *CV); @@ -400,7 +403,7 @@ public: /// By default, this method prints the label for the specified /// MachineBasicBlock, an alignment (if present) and a comment describing it /// if appropriate. - virtual void EmitBasicBlockStart(const MachineBasicBlock &MBB) const; + virtual void EmitBasicBlockStart(const MachineBasicBlock &MBB); /// Targets can override this to emit stuff at the end of a basic block. virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB); @@ -415,6 +418,10 @@ public: virtual void EmitFunctionEntryLabel(); + virtual void EmitFunctionDescriptor() { + llvm_unreachable("Function descriptor is target-specific."); + } + virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV); /// Targets can override this to change how global constants that are part of @@ -635,6 +642,10 @@ public: /// supported by the target. void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const; + /// Return the alignment for the specified \p GV. + static Align getGVAlignment(const GlobalValue *GV, const DataLayout &DL, + Align InAlign = Align::None()); + private: /// Private state for PrintSpecial() // Assign a unique ID to this machine instruction. diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h index 70bf670fdf0b..2e57b4c9d332 100644 --- a/include/llvm/CodeGen/BasicTTIImpl.h +++ b/include/llvm/CodeGen/BasicTTIImpl.h @@ -190,6 +190,7 @@ private: protected: explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL) : BaseT(DL) {} + virtual ~BasicTTIImplBase() = default; using TargetTransformInfoImplBase::DL; @@ -215,6 +216,16 @@ public: return -1; } + bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, + Intrinsic::ID IID) const { + return false; + } + + bool rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, + Value *OldV, Value *NewV) const { + return false; + } + bool isLegalAddImmediate(int64_t imm) { return getTLI()->isLegalAddImmediate(imm); } @@ -317,7 +328,7 @@ public: unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JumpTableSize) { /// Try to find the estimated number of clusters. Note that the number of - /// clusters identified in this function could be different from the actural + /// clusters identified in this function could be different from the actual /// numbers found in lowering. This function ignore switches that are /// lowered with a mix of jump table / bit test / BTree. This function was /// initially intended to be used when estimating the cost of switch in @@ -371,10 +382,6 @@ public: return N; } - unsigned getJumpBufAlignment() { return getTLI()->getJumpBufAlignment(); } - - unsigned getJumpBufSize() { return getTLI()->getJumpBufSize(); } - bool shouldBuildLookupTables() { const TargetLoweringBase *TLI = getTLI(); return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || @@ -508,13 +515,44 @@ public: return BaseT::getInstructionLatency(I); } + virtual Optional<unsigned> + getCacheSize(TargetTransformInfo::CacheLevel Level) const { + return Optional<unsigned>( + getST()->getCacheSize(static_cast<unsigned>(Level))); + } + + virtual Optional<unsigned> + getCacheAssociativity(TargetTransformInfo::CacheLevel Level) const { + Optional<unsigned> TargetResult = + getST()->getCacheAssociativity(static_cast<unsigned>(Level)); + + if (TargetResult) + return TargetResult; + + return BaseT::getCacheAssociativity(Level); + } + + virtual unsigned getCacheLineSize() const { + return getST()->getCacheLineSize(); + } + + virtual unsigned getPrefetchDistance() const { + return getST()->getPrefetchDistance(); + } + + virtual unsigned getMinPrefetchStride() const { + return getST()->getMinPrefetchStride(); + } + + virtual unsigned getMaxPrefetchIterationsAhead() const { + return getST()->getMaxPrefetchIterationsAhead(); + } + /// @} /// \name Vector TTI Implementations /// @{ - unsigned getNumberOfRegisters(bool Vector) { return Vector ? 0 : 1; } - unsigned getRegisterBitWidth(bool Vector) const { return 32; } /// Estimate the overhead of scalarizing an instruction. Insert and Extract @@ -1111,9 +1149,7 @@ public: OpPropsBW); // For non-rotates (X != Y) we must add shift-by-zero handling costs. if (X != Y) { - Type *CondTy = Type::getInt1Ty(RetTy->getContext()); - if (RetVF > 1) - CondTy = VectorType::get(CondTy, RetVF); + Type *CondTy = RetTy->getWithNewBitWidth(1); Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy, nullptr); Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy, @@ -1131,7 +1167,6 @@ public: unsigned getIntrinsicInstrCost( Intrinsic::ID IID, Type *RetTy, ArrayRef<Type *> Tys, FastMathFlags FMF, unsigned ScalarizationCostPassed = std::numeric_limits<unsigned>::max()) { - unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1); auto *ConcreteTTI = static_cast<T *>(this); SmallVector<unsigned, 2> ISDs; @@ -1288,9 +1323,7 @@ public: /*IsUnsigned=*/false); case Intrinsic::sadd_sat: case Intrinsic::ssub_sat: { - Type *CondTy = Type::getInt1Ty(RetTy->getContext()); - if (RetVF > 1) - CondTy = VectorType::get(CondTy, RetVF); + Type *CondTy = RetTy->getWithNewBitWidth(1); Type *OpTy = StructType::create({RetTy, CondTy}); Intrinsic::ID OverflowOp = IID == Intrinsic::sadd_sat @@ -1310,9 +1343,7 @@ public: } case Intrinsic::uadd_sat: case Intrinsic::usub_sat: { - Type *CondTy = Type::getInt1Ty(RetTy->getContext()); - if (RetVF > 1) - CondTy = VectorType::get(CondTy, RetVF); + Type *CondTy = RetTy->getWithNewBitWidth(1); Type *OpTy = StructType::create({RetTy, CondTy}); Intrinsic::ID OverflowOp = IID == Intrinsic::uadd_sat @@ -1329,9 +1360,7 @@ public: case Intrinsic::smul_fix: case Intrinsic::umul_fix: { unsigned ExtSize = RetTy->getScalarSizeInBits() * 2; - Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize); - if (RetVF > 1) - ExtTy = VectorType::get(ExtTy, RetVF); + Type *ExtTy = RetTy->getWithNewBitWidth(ExtSize); unsigned ExtOp = IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt; @@ -1395,9 +1424,7 @@ public: Type *MulTy = RetTy->getContainedType(0); Type *OverflowTy = RetTy->getContainedType(1); unsigned ExtSize = MulTy->getScalarSizeInBits() * 2; - Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize); - if (MulTy->isVectorTy()) - ExtTy = VectorType::get(ExtTy, MulTy->getVectorNumElements() ); + Type *ExtTy = MulTy->getWithNewBitWidth(ExtSize); unsigned ExtOp = IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt; diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index aa339e1cc913..a30ca638ee6d 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -20,6 +20,7 @@ #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallingConv.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/Alignment.h" namespace llvm { @@ -43,6 +44,7 @@ public: AExtUpper, // The value is in the upper bits of the location and should be // extended with undefined upper bits when retrieved. BCvt, // The value is bit-converted in the location. + Trunc, // The value is truncated in the location. VExt, // The value is vector-widened in the location. // FIXME: Not implemented yet. Code that uses AExt to mean // vector-widen should be fixed to use VExt instead. @@ -197,7 +199,7 @@ private: LLVMContext &Context; unsigned StackOffset; - unsigned MaxStackArgAlign; + Align MaxStackArgAlign; SmallVector<uint32_t, 16> UsedRegs; SmallVector<CCValAssign, 4> PendingLocs; SmallVector<ISD::ArgFlagsTy, 4> PendingArgFlags; @@ -421,19 +423,19 @@ public: /// AllocateStack - Allocate a chunk of stack space with the specified size /// and alignment. - unsigned AllocateStack(unsigned Size, unsigned Align) { - assert(Align && ((Align - 1) & Align) == 0); // Align is power of 2. - StackOffset = alignTo(StackOffset, Align); + unsigned AllocateStack(unsigned Size, unsigned Alignment) { + const Align CheckedAlignment(Alignment); + StackOffset = alignTo(StackOffset, CheckedAlignment); unsigned Result = StackOffset; StackOffset += Size; - MaxStackArgAlign = std::max(Align, MaxStackArgAlign); - ensureMaxAlignment(Align); + MaxStackArgAlign = std::max(CheckedAlignment, MaxStackArgAlign); + ensureMaxAlignment(CheckedAlignment); return Result; } - void ensureMaxAlignment(unsigned Align) { + void ensureMaxAlignment(Align Alignment) { if (!AnalyzingMustTailForwardedRegs) - MF.getFrameInfo().ensureMaxAlignment(Align); + MF.getFrameInfo().ensureMaxAlignment(Alignment.value()); } /// Version of AllocateStack with extra register to be shadowed. diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h index cf58ee0cabea..705465b15c4c 100644 --- a/include/llvm/CodeGen/DFAPacketizer.h +++ b/include/llvm/CodeGen/DFAPacketizer.h @@ -28,6 +28,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAGMutation.h" +#include "llvm/Support/Automaton.h" #include <cstdint> #include <map> #include <memory> @@ -76,26 +77,26 @@ using DFAStateInput = int64_t; class DFAPacketizer { private: - using UnsignPair = std::pair<unsigned, DFAInput>; - const InstrItineraryData *InstrItins; - int CurrentState = 0; - const DFAStateInput (*DFAStateInputTable)[2]; - const unsigned *DFAStateEntryTable; - - // CachedTable is a map from <FromState, Input> to ToState. - DenseMap<UnsignPair, unsigned> CachedTable; - - // Read the DFA transition table and update CachedTable. - void ReadTable(unsigned state); + Automaton<DFAInput> A; public: - DFAPacketizer(const InstrItineraryData *I, const DFAStateInput (*SIT)[2], - const unsigned *SET); + DFAPacketizer(const InstrItineraryData *InstrItins, Automaton<uint64_t> a) : + InstrItins(InstrItins), A(std::move(a)) { + // Start off with resource tracking disabled. + A.enableTranscription(false); + } // Reset the current state to make all resources available. void clearResources() { - CurrentState = 0; + A.reset(); + } + + // Set whether this packetizer should track not just whether instructions + // can be packetized, but also which functional units each instruction ends up + // using after packetization. + void setTrackResources(bool Track) { + A.enableTranscription(Track); } // Return the DFAInput for an instruction class. @@ -120,6 +121,15 @@ public: // current state to reflect that change. void reserveResources(MachineInstr &MI); + // Return the resources used by the InstIdx'th instruction added to this + // packet. The resources are returned as a bitvector of functional units. + // + // Note that a bundle may be packed in multiple valid ways. This function + // returns one arbitary valid packing. + // + // Requires setTrackResources(true) to have been called. + unsigned getUsedResources(unsigned InstIdx); + const InstrItineraryData *getInstrItins() const { return InstrItins; } }; @@ -134,7 +144,7 @@ class VLIWPacketizerList { protected: MachineFunction &MF; const TargetInstrInfo *TII; - AliasAnalysis *AA; + AAResults *AA; // The VLIW Scheduler. DefaultVLIWScheduler *VLIWScheduler; @@ -146,9 +156,9 @@ protected: std::map<MachineInstr*, SUnit*> MIToSUnit; public: - // The AliasAnalysis parameter can be nullptr. + // The AAResults parameter can be nullptr. VLIWPacketizerList(MachineFunction &MF, MachineLoopInfo &MLI, - AliasAnalysis *AA); + AAResults *AA); virtual ~VLIWPacketizerList(); diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h index 684f9e40ca5a..e8e7504a6cda 100644 --- a/include/llvm/CodeGen/DIE.h +++ b/include/llvm/CodeGen/DIE.h @@ -550,6 +550,14 @@ public: return *static_cast<T *>(Last ? Last->Next.getPointer() : nullptr); } + void takeNodes(IntrusiveBackList<T> &Other) { + for (auto &N : Other) { + N.Next.setPointerAndInt(&N, true); + push_back(N); + } + Other.Last = nullptr; + } + class const_iterator; class iterator : public iterator_facade_base<iterator, std::forward_iterator_tag, T> { @@ -685,6 +693,10 @@ public: return addValue(Alloc, DIEValue(Attribute, Form, std::forward<T>(Value))); } + /// Take ownership of the nodes in \p Other, and append them to the back of + /// the list. + void takeValues(DIEValueList &Other) { List.takeNodes(Other.List); } + value_range values() { return make_range(value_iterator(List.begin()), value_iterator(List.end())); } diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h index f09b59daf4dd..03d681feb7aa 100644 --- a/include/llvm/CodeGen/FastISel.h +++ b/include/llvm/CodeGen/FastISel.h @@ -93,9 +93,9 @@ public: SmallVector<Value *, 16> OutVals; SmallVector<ISD::ArgFlagsTy, 16> OutFlags; - SmallVector<unsigned, 16> OutRegs; + SmallVector<Register, 16> OutRegs; SmallVector<ISD::InputArg, 4> Ins; - SmallVector<unsigned, 4> InRegs; + SmallVector<Register, 4> InRegs; CallLoweringInfo() : RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false), diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index fb60191abd3a..f812a2f6c585 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -20,7 +20,6 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -37,6 +36,7 @@ namespace llvm { class Argument; class BasicBlock; class BranchProbabilityInfo; +class LegacyDivergenceAnalysis; class Function; class Instruction; class MachineFunction; diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h index d717121ad78e..4901a3748e4a 100644 --- a/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -45,18 +45,62 @@ class CallLowering { public: struct ArgInfo { SmallVector<Register, 4> Regs; + // If the argument had to be split into multiple parts according to the + // target calling convention, then this contains the original vregs + // if the argument was an incoming arg. + SmallVector<Register, 2> OrigRegs; Type *Ty; - ISD::ArgFlagsTy Flags; + SmallVector<ISD::ArgFlagsTy, 4> Flags; bool IsFixed; ArgInfo(ArrayRef<Register> Regs, Type *Ty, - ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, bool IsFixed = true) - : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags), - IsFixed(IsFixed) { + ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), + bool IsFixed = true) + : Regs(Regs.begin(), Regs.end()), Ty(Ty), + Flags(Flags.begin(), Flags.end()), IsFixed(IsFixed) { + if (!Regs.empty() && Flags.empty()) + this->Flags.push_back(ISD::ArgFlagsTy()); // FIXME: We should have just one way of saying "no register". assert((Ty->isVoidTy() == (Regs.empty() || Regs[0] == 0)) && "only void types should have no register"); } + + ArgInfo() : Ty(nullptr), IsFixed(false) {} + }; + + struct CallLoweringInfo { + /// Calling convention to be used for the call. + CallingConv::ID CallConv = CallingConv::C; + + /// Destination of the call. It should be either a register, globaladdress, + /// or externalsymbol. + MachineOperand Callee = MachineOperand::CreateImm(0); + + /// Descriptor for the return type of the function. + ArgInfo OrigRet; + + /// List of descriptors of the arguments passed to the function. + SmallVector<ArgInfo, 8> OrigArgs; + + /// Valid if the call has a swifterror inout parameter, and contains the + /// vreg that the swifterror should be copied into after the call. + Register SwiftErrorVReg = 0; + + MDNode *KnownCallees = nullptr; + + /// True if the call must be tail call optimized. + bool IsMustTailCall = false; + + /// True if the call passes all target-independent checks for tail call + /// optimization. + bool IsTailCall = false; + + /// True if the call was lowered as a tail call. This is consumed by the + /// legalizer. This allows the legalizer to lower libcalls as tail calls. + bool LoweredTailCall = false; + + /// True if the call is to a vararg function. + bool IsVarArg = false; }; /// Argument handling is mostly uniform between the four places that @@ -72,9 +116,9 @@ public: virtual ~ValueHandler() = default; - /// Returns true if the handler is dealing with formal arguments, - /// not with return values etc. - virtual bool isArgumentHandler() const { return false; } + /// Returns true if the handler is dealing with incoming arguments, + /// i.e. those that move values from some physical location to vregs. + virtual bool isIncomingArgumentHandler() const = 0; /// Materialize a VReg containing the address of the specified /// stack-based object. This is either based on a FrameIndex or @@ -112,8 +156,8 @@ public: virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, const ArgInfo &Info, - CCState &State) { - return AssignFn(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State); + ISD::ArgFlagsTy Flags, CCState &State) { + return AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); } MachineIRBuilder &MIRBuilder; @@ -162,12 +206,42 @@ protected: /// \p Callback to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. - bool handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args, + bool handleAssignments(MachineIRBuilder &MIRBuilder, + SmallVectorImpl<ArgInfo> &Args, ValueHandler &Handler) const; bool handleAssignments(CCState &CCState, SmallVectorImpl<CCValAssign> &ArgLocs, - MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args, + MachineIRBuilder &MIRBuilder, + SmallVectorImpl<ArgInfo> &Args, ValueHandler &Handler) const; + + /// Analyze passed or returned values from a call, supplied in \p ArgInfo, + /// incorporating info about the passed values into \p CCState. + /// + /// Used to check if arguments are suitable for tail call lowering. + bool analyzeArgInfo(CCState &CCState, SmallVectorImpl<ArgInfo> &Args, + CCAssignFn &AssignFnFixed, + CCAssignFn &AssignFnVarArg) const; + + /// \returns True if the calling convention for a callee and its caller pass + /// results in the same way. Typically used for tail call eligibility checks. + /// + /// \p Info is the CallLoweringInfo for the call. + /// \p MF is the MachineFunction for the caller. + /// \p InArgs contains the results of the call. + /// \p CalleeAssignFnFixed is the CCAssignFn to be used for the callee for + /// fixed arguments. + /// \p CalleeAssignFnVarArg is similar, but for varargs. + /// \p CallerAssignFnFixed is the CCAssignFn to be used for the caller for + /// fixed arguments. + /// \p CallerAssignFnVarArg is similar, but for varargs. + bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, + SmallVectorImpl<ArgInfo> &InArgs, + CCAssignFn &CalleeAssignFnFixed, + CCAssignFn &CalleeAssignFnVarArg, + CCAssignFn &CallerAssignFnFixed, + CCAssignFn &CallerAssignFnVarArg) const; + public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} virtual ~CallLowering() = default; @@ -223,37 +297,10 @@ public: /// This hook must be implemented to lower the given call instruction, /// including argument and return value marshalling. /// - /// \p CallConv is the calling convention to be used for the call. - /// - /// \p Callee is the destination of the call. It should be either a register, - /// globaladdress, or externalsymbol. - /// - /// \p OrigRet is a descriptor for the return type of the function. - /// - /// \p OrigArgs is a list of descriptors of the arguments passed to the - /// function. - /// - /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout - /// parameter, and contains the vreg that the swifterror should be copied into - /// after the call. /// /// \return true if the lowering succeeded, false otherwise. - virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, - const MachineOperand &Callee, const ArgInfo &OrigRet, - ArrayRef<ArgInfo> OrigArgs, - Register SwiftErrorVReg) const { - if (!supportSwiftError()) { - assert(SwiftErrorVReg == 0 && "trying to use unsupported swifterror"); - return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs); - } - return false; - } - - /// This hook behaves as the extended lowerCall function, but for targets that - /// do not support swifterror value promotion. - virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, - const MachineOperand &Callee, const ArgInfo &OrigRet, - ArrayRef<ArgInfo> OrigArgs) const { + virtual bool lowerCall(MachineIRBuilder &MIRBuilder, + CallLoweringInfo &Info) const { return false; } diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 0c50c9c5e0cf..4c04dc52547d 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -27,6 +27,8 @@ class MachineIRBuilder; class MachineRegisterInfo; class MachineInstr; class MachineOperand; +class GISelKnownBits; +class MachineDominatorTree; struct PreferredTuple { LLT Ty; // The result type of the extend. @@ -35,12 +37,17 @@ struct PreferredTuple { }; class CombinerHelper { +protected: MachineIRBuilder &Builder; MachineRegisterInfo &MRI; GISelChangeObserver &Observer; + GISelKnownBits *KB; + MachineDominatorTree *MDT; public: - CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B); + CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, + GISelKnownBits *KB = nullptr, + MachineDominatorTree *MDT = nullptr); /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const; @@ -56,18 +63,132 @@ public: bool matchCombineCopy(MachineInstr &MI); void applyCombineCopy(MachineInstr &MI); + /// Returns true if \p DefMI precedes \p UseMI or they are the same + /// instruction. Both must be in the same basic block. + bool isPredecessor(MachineInstr &DefMI, MachineInstr &UseMI); + + /// Returns true if \p DefMI dominates \p UseMI. By definition an + /// instruction dominates itself. + /// + /// If we haven't been provided with a MachineDominatorTree during + /// construction, this function returns a conservative result that tracks just + /// a single basic block. + bool dominates(MachineInstr &DefMI, MachineInstr &UseMI); + /// If \p MI is extend that consumes the result of a load, try to combine it. /// Returns true if MI changed. bool tryCombineExtendingLoads(MachineInstr &MI); bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); - bool matchCombineBr(MachineInstr &MI); - bool tryCombineBr(MachineInstr &MI); + /// Combine \p MI into a pre-indexed or post-indexed load/store operation if + /// legal and the surrounding code makes it useful. + bool tryCombineIndexedLoadStore(MachineInstr &MI); + + bool matchElideBrByInvertingCond(MachineInstr &MI); + void applyElideBrByInvertingCond(MachineInstr &MI); + bool tryElideBrByInvertingCond(MachineInstr &MI); + + /// If \p MI is G_CONCAT_VECTORS, try to combine it. + /// Returns true if MI changed. + /// Right now, we support: + /// - concat_vector(undef, undef) => undef + /// - concat_vector(build_vector(A, B), build_vector(C, D)) => + /// build_vector(A, B, C, D) + /// + /// \pre MI.getOpcode() == G_CONCAT_VECTORS. + bool tryCombineConcatVectors(MachineInstr &MI); + /// Check if the G_CONCAT_VECTORS \p MI is undef or if it + /// can be flattened into a build_vector. + /// In the first case \p IsUndef will be true. + /// In the second case \p Ops will contain the operands needed + /// to produce the flattened build_vector. + /// + /// \pre MI.getOpcode() == G_CONCAT_VECTORS. + bool matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef, + SmallVectorImpl<Register> &Ops); + /// Replace \p MI with a flattened build_vector with \p Ops or an + /// implicit_def if IsUndef is true. + void applyCombineConcatVectors(MachineInstr &MI, bool IsUndef, + const ArrayRef<Register> Ops); + + /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS. + /// Returns true if MI changed. + /// + /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR. + bool tryCombineShuffleVector(MachineInstr &MI); + /// Check if the G_SHUFFLE_VECTOR \p MI can be replaced by a + /// concat_vectors. + /// \p Ops will contain the operands needed to produce the flattened + /// concat_vectors. + /// + /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR. + bool matchCombineShuffleVector(MachineInstr &MI, + SmallVectorImpl<Register> &Ops); + /// Replace \p MI with a concat_vectors with \p Ops. + void applyCombineShuffleVector(MachineInstr &MI, + const ArrayRef<Register> Ops); + + /// Optimize memcpy intrinsics et al, e.g. constant len calls. + /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline. + /// + /// For example (pre-indexed): + /// + /// $addr = G_GEP $base, $offset + /// [...] + /// $val = G_LOAD $addr + /// [...] + /// $whatever = COPY $addr + /// + /// --> + /// + /// $val, $addr = G_INDEXED_LOAD $base, $offset, 1 (IsPre) + /// [...] + /// $whatever = COPY $addr + /// + /// or (post-indexed): + /// + /// G_STORE $val, $base + /// [...] + /// $addr = G_GEP $base, $offset + /// [...] + /// $whatever = COPY $addr + /// + /// --> + /// + /// $addr = G_INDEXED_STORE $val, $base, $offset + /// [...] + /// $whatever = COPY $addr + bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); + +private: + // Memcpy family optimization helpers. + bool optimizeMemcpy(MachineInstr &MI, Register Dst, Register Src, + unsigned KnownLen, unsigned DstAlign, unsigned SrcAlign, + bool IsVolatile); + bool optimizeMemmove(MachineInstr &MI, Register Dst, Register Src, + unsigned KnownLen, unsigned DstAlign, unsigned SrcAlign, + bool IsVolatile); + bool optimizeMemset(MachineInstr &MI, Register Dst, Register Val, + unsigned KnownLen, unsigned DstAlign, bool IsVolatile); + + /// Given a non-indexed load or store instruction \p MI, find an offset that + /// can be usefully and legally folded into it as a post-indexing operation. + /// + /// \returns true if a candidate is found. + bool findPostIndexCandidate(MachineInstr &MI, Register &Addr, Register &Base, + Register &Offset); + + /// Given a non-indexed load or store instruction \p MI, find an offset that + /// can be usefully and legally folded into it as a pre-indexing operation. + /// + /// \returns true if a candidate is found. + bool findPreIndexCandidate(MachineInstr &MI, Register &Addr, Register &Base, + Register &Offset); }; } // namespace llvm diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index 3b09a8e2b479..ad645a46bbe6 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -27,9 +27,11 @@ class MachineRegisterInfo; class CombinerInfo { public: CombinerInfo(bool AllowIllegalOps, bool ShouldLegalizeIllegal, - LegalizerInfo *LInfo) + LegalizerInfo *LInfo, bool OptEnabled, bool OptSize, + bool MinSize) : IllegalOpsAllowed(AllowIllegalOps), - LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo) { + LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo), + EnableOpt(OptEnabled), EnableOptSize(OptSize), EnableMinSize(MinSize) { assert(((AllowIllegalOps || !LegalizeIllegalOps) || LInfo) && "Expecting legalizerInfo when illegalops not allowed"); } @@ -43,6 +45,15 @@ public: bool LegalizeIllegalOps; // TODO: Make use of this. const LegalizerInfo *LInfo; + /// Whether optimizations should be enabled. This is to distinguish between + /// uses of the combiner unconditionally and only when optimizations are + /// specifically enabled/ + bool EnableOpt; + /// Whether we're optimizing for size. + bool EnableOptSize; + /// Whether we're optimizing for minsize (-Oz). + bool EnableMinSize; + /// Attempt to combine instructions using MI as the root. /// /// Use Observer to report the creation, modification, and erasure of diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h index e817d9b4550e..df196bfbd437 100644 --- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -54,6 +54,17 @@ public: return buildConstant(Dst, MaybeCst->getSExtValue()); break; } + case TargetOpcode::G_SEXT_INREG: { + assert(DstOps.size() == 1 && "Invalid dst ops"); + assert(SrcOps.size() == 2 && "Invalid src ops"); + const DstOp &Dst = DstOps[0]; + const SrcOp &Src0 = SrcOps[0]; + const SrcOp &Src1 = SrcOps[1]; + if (auto MaybeCst = + ConstantFoldExtOp(Opc, Src0.getReg(), Src1.getImm(), *getMRI())) + return buildConstant(Dst, MaybeCst->getSExtValue()); + break; + } } return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps); } diff --git a/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h new file mode 100644 index 000000000000..dfe5a7f3177d --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h @@ -0,0 +1,111 @@ +//===- llvm/CodeGen/GlobalISel/GISelKnownBits.h ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// Provides analysis for querying information about KnownBits during GISel +/// passes. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H +#define LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H + +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/Register.h" +#include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/KnownBits.h" + +namespace llvm { + +class TargetLowering; +class DataLayout; + +class GISelKnownBits : public GISelChangeObserver { + MachineFunction &MF; + MachineRegisterInfo &MRI; + const TargetLowering &TL; + const DataLayout &DL; + +public: + GISelKnownBits(MachineFunction &MF); + virtual ~GISelKnownBits() = default; + void setMF(MachineFunction &MF); + virtual void computeKnownBitsImpl(Register R, KnownBits &Known, + const APInt &DemandedElts, + unsigned Depth = 0); + + // KnownBitsAPI + KnownBits getKnownBits(Register R); + // Calls getKnownBits for first operand def of MI. + KnownBits getKnownBits(MachineInstr &MI); + APInt getKnownZeroes(Register R); + APInt getKnownOnes(Register R); + + /// \return true if 'V & Mask' is known to be zero in DemandedElts. We use + /// this predicate to simplify operations downstream. + /// Mask is known to be zero for bits that V cannot have. + bool maskedValueIsZero(Register Val, const APInt &Mask) { + return Mask.isSubsetOf(getKnownBits(Val).Zero); + } + + /// \return true if the sign bit of Op is known to be zero. We use this + /// predicate to simplify operations downstream. + bool signBitIsZero(Register Op); + + // FIXME: Is this the right place for G_FRAME_INDEX? Should it be in + // TargetLowering? + void computeKnownBitsForFrameIndex(Register R, KnownBits &Known, + const APInt &DemandedElts, + unsigned Depth = 0); + static Align inferAlignmentForFrameIdx(int FrameIdx, int Offset, + const MachineFunction &MF); + static void computeKnownBitsForAlignment(KnownBits &Known, + MaybeAlign Alignment); + + // Try to infer alignment for MI. + static MaybeAlign inferPtrAlignment(const MachineInstr &MI); + + // Observer API. No-op for non-caching implementation. + void erasingInstr(MachineInstr &MI) override{}; + void createdInstr(MachineInstr &MI) override{}; + void changingInstr(MachineInstr &MI) override{}; + void changedInstr(MachineInstr &MI) override{}; + +protected: + unsigned getMaxDepth() const { return 6; } +}; + +/// To use KnownBitsInfo analysis in a pass, +/// KnownBitsInfo &Info = getAnalysis<GISelKnownBitsInfoAnalysis>().get(MF); +/// Add to observer if the Info is caching. +/// WrapperObserver.addObserver(Info); + +/// Eventually add other features such as caching/ser/deserializing +/// to MIR etc. Those implementations can derive from GISelKnownBits +/// and override computeKnownBitsImpl. +class GISelKnownBitsAnalysis : public MachineFunctionPass { + std::unique_ptr<GISelKnownBits> Info; + +public: + static char ID; + GISelKnownBitsAnalysis() : MachineFunctionPass(ID) { + initializeGISelKnownBitsAnalysisPass(*PassRegistry::getPassRegistry()); + } + GISelKnownBits &get(MachineFunction &MF) { + if (!Info) + Info = std::make_unique<GISelKnownBits>(MF); + return *Info.get(); + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnMachineFunction(MachineFunction &MF) override; + void releaseMemory() override { Info.reset(); } +}; +} // namespace llvm + +#endif // ifdef diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 8654ba83f08d..bdb92aa4689d 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -213,8 +213,8 @@ private: bool translateStore(const User &U, MachineIRBuilder &MIRBuilder); /// Translate an LLVM string intrinsic (memcpy, memset, ...). - bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, - unsigned ID); + bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, + Intrinsic::ID ID); void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); @@ -243,6 +243,10 @@ private: bool valueIsSplit(const Value &V, SmallVectorImpl<uint64_t> *Offsets = nullptr); + /// Common code for translating normal calls or invokes. + bool translateCallSite(const ImmutableCallSite &CS, + MachineIRBuilder &MIRBuilder); + /// Translate call instruction. /// \pre \p U is a call instruction. bool translateCall(const User &U, MachineIRBuilder &MIRBuilder); @@ -514,6 +518,10 @@ private: // function has the optnone attribute. bool EnableOpts = false; + /// True when the block contains a tail call. This allows the IRTranslator to + /// stop translating such blocks early. + bool HasTailCall = false; + /// Switch analysis and optimization. class GISelSwitchLowering : public SwitchCG::SwitchLowering { public: diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index e9b93be76754..fd3dc743000b 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -31,6 +31,7 @@ namespace llvm { class APInt; class APFloat; +class GISelKnownBits; class MachineInstr; class MachineInstrBuilder; class MachineFunction; @@ -148,6 +149,13 @@ enum { /// - AddrSpaceN+1 ... GIM_CheckMemoryAddressSpace, + /// Check the minimum alignment of the memory access for the given machine + /// memory operand. + /// - InsnID - Instruction ID + /// - MMOIdx - MMO index + /// - MinAlign - Minimum acceptable alignment + GIM_CheckMemoryAlignment, + /// Check the size of the memory access for the given machine memory operand /// against the size of an operand. /// - InsnID - Instruction ID @@ -201,11 +209,22 @@ enum { /// - Expected Intrinsic ID GIM_CheckIntrinsicID, + /// Check the operand is a specific predicate + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected predicate + GIM_CheckCmpPredicate, + /// Check the specified operand is an MBB /// - InsnID - Instruction ID /// - OpIdx - Operand index GIM_CheckIsMBB, + /// Check the specified operand is an Imm + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + GIM_CheckIsImm, + /// Check if the specified operand is safe to fold into the current /// instruction. /// - InsnID - Instruction ID @@ -365,7 +384,20 @@ public: /// if returns true: /// for I in all mutated/inserted instructions: /// !isPreISelGenericOpcode(I.getOpcode()) - virtual bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const = 0; + virtual bool select(MachineInstr &I) = 0; + + CodeGenCoverage *CoverageInfo = nullptr; + GISelKnownBits *KnownBits = nullptr; + MachineFunction *MF = nullptr; + + /// Setup per-MF selector state. + virtual void setupMF(MachineFunction &mf, + GISelKnownBits &KB, + CodeGenCoverage &covinfo) { + CoverageInfo = &covinfo; + KnownBits = &KB; + MF = &mf; + } protected: using ComplexRendererFns = diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index e8ee4af0cb0b..08f2f54bcf90 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -98,7 +98,7 @@ bool InstructionSelector::executeMatchTable( return false; break; } - if (TRI.isPhysicalRegister(MO.getReg())) { + if (Register::isPhysicalRegister(MO.getReg())) { DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": Is a physical register\n"); if (handleReject() == RejectAndGiveUp) @@ -409,6 +409,30 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckMemoryAlignment: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t MMOIdx = MatchTable[CurrentIdx++]; + unsigned MinAlign = MatchTable[CurrentIdx++]; + + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + MachineMemOperand *MMO + = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" + << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx + << ")->getAlignment() >= " << MinAlign << ")\n"); + if (MMO->getAlignment() < MinAlign && handleReject() == RejectAndGiveUp) + return false; + + break; + } case GIM_CheckMemorySizeEqualTo: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t MMOIdx = MatchTable[CurrentIdx++]; @@ -638,7 +662,21 @@ bool InstructionSelector::executeMatchTable( return false; break; } - + case GIM_CheckCmpPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isPredicate() || MO.getPredicate() != Value) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckIsMBB: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; @@ -652,7 +690,19 @@ bool InstructionSelector::executeMatchTable( } break; } - + case GIM_CheckIsImm: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID + << "]->getOperand(" << OpIdx << "))\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } case GIM_CheckIsSafeToFold: { int64_t InsnID = MatchTable[CurrentIdx++]; DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), @@ -792,11 +842,13 @@ bool InstructionSelector::executeMatchTable( case GIR_AddRegister: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t RegNum = MatchTable[CurrentIdx++]; + uint64_t RegFlags = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - OutMIs[InsnID].addReg(RegNum); - DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), - dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" - << InsnID << "], " << RegNum << ")\n"); + OutMIs[InsnID].addReg(RegNum, RegFlags); + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" + << InsnID << "], " << RegNum << ", " << RegFlags << ")\n"); break; } diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index a22778b8848c..7f960e727846 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -47,8 +47,7 @@ public: bool tryCombineAnyExt(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { - if (MI.getOpcode() != TargetOpcode::G_ANYEXT) - return false; + assert(MI.getOpcode() == TargetOpcode::G_ANYEXT); Builder.setInstr(MI); Register DstReg = MI.getOperand(0).getReg(); @@ -93,9 +92,7 @@ public: bool tryCombineZExt(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { - - if (MI.getOpcode() != TargetOpcode::G_ZEXT) - return false; + assert(MI.getOpcode() == TargetOpcode::G_ZEXT); Builder.setInstr(MI); Register DstReg = MI.getOperand(0).getReg(); @@ -136,32 +133,24 @@ public: bool tryCombineSExt(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { - - if (MI.getOpcode() != TargetOpcode::G_SEXT) - return false; + assert(MI.getOpcode() == TargetOpcode::G_SEXT); Builder.setInstr(MI); Register DstReg = MI.getOperand(0).getReg(); Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); - // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c + // sext(trunc x) - > (sext_inreg (aext/copy/trunc x), c) Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); - // Guess on the RHS shift amount type, which should be re-legalized if - // applicable. - if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy, DstTy}}) || - isInstUnsupported({TargetOpcode::G_ASHR, {DstTy, DstTy}}) || - isConstantUnsupported(DstTy)) + if (isInstUnsupported({TargetOpcode::G_SEXT_INREG, {DstTy}})) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); - unsigned ShAmt = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits(); - auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt); - auto MIBShl = Builder.buildInstr( - TargetOpcode::G_SHL, {DstTy}, - {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBShAmt}); - Builder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {MIBShl, MIBShAmt}); + uint64_t SizeInBits = SrcTy.getScalarSizeInBits(); + Builder.buildInstr( + TargetOpcode::G_SEXT_INREG, {DstReg}, + {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), SizeInBits}); markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } @@ -172,9 +161,8 @@ public: bool tryFoldImplicitDef(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { unsigned Opcode = MI.getOpcode(); - if (Opcode != TargetOpcode::G_ANYEXT && Opcode != TargetOpcode::G_ZEXT && - Opcode != TargetOpcode::G_SEXT) - return false; + assert(Opcode == TargetOpcode::G_ANYEXT || Opcode == TargetOpcode::G_ZEXT || + Opcode == TargetOpcode::G_SEXT); if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(), MRI)) { @@ -203,21 +191,38 @@ public: return false; } - static unsigned getMergeOpcode(LLT OpTy, LLT DestTy) { + static unsigned canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp, + LLT OpTy, LLT DestTy) { if (OpTy.isVector() && DestTy.isVector()) - return TargetOpcode::G_CONCAT_VECTORS; + return MergeOp == TargetOpcode::G_CONCAT_VECTORS; + + if (OpTy.isVector() && !DestTy.isVector()) { + if (MergeOp == TargetOpcode::G_BUILD_VECTOR) + return true; - if (OpTy.isVector() && !DestTy.isVector()) - return TargetOpcode::G_BUILD_VECTOR; + if (MergeOp == TargetOpcode::G_CONCAT_VECTORS) { + if (ConvertOp == 0) + return true; - return TargetOpcode::G_MERGE_VALUES; + const unsigned OpEltSize = OpTy.getElementType().getSizeInBits(); + + // Don't handle scalarization with a cast that isn't in the same + // direction as the vector cast. This could be handled, but it would + // require more intermediate unmerges. + if (ConvertOp == TargetOpcode::G_TRUNC) + return DestTy.getSizeInBits() <= OpEltSize; + return DestTy.getSizeInBits() >= OpEltSize; + } + + return false; + } + + return MergeOp == TargetOpcode::G_MERGE_VALUES; } bool tryCombineMerges(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { - - if (MI.getOpcode() != TargetOpcode::G_UNMERGE_VALUES) - return false; + assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); unsigned NumDefs = MI.getNumOperands() - 1; MachineInstr *SrcDef = @@ -237,16 +242,14 @@ public: MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI); } - // FIXME: Handle scalarizing concat_vectors (scalar result type with vector - // source) - unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy); - if (!MergeI || MergeI->getOpcode() != MergingOpcode) + if (!MergeI || !canFoldMergeOpcode(MergeI->getOpcode(), + ConvertOp, OpTy, DestTy)) return false; const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; if (NumMergeRegs < NumDefs) { - if (ConvertOp != 0 || NumDefs % NumMergeRegs != 0) + if (NumDefs % NumMergeRegs != 0) return false; Builder.setInstr(MI); @@ -264,7 +267,22 @@ public: ++j, ++DefIdx) DstRegs.push_back(MI.getOperand(DefIdx).getReg()); - Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg()); + if (ConvertOp) { + SmallVector<Register, 2> TmpRegs; + // This is a vector that is being scalarized and casted. Extract to + // the element type, and do the conversion on the scalars. + LLT MergeEltTy + = MRI.getType(MergeI->getOperand(0).getReg()).getElementType(); + for (unsigned j = 0; j < NumMergeRegs; ++j) + TmpRegs.push_back(MRI.createGenericVirtualRegister(MergeEltTy)); + + Builder.buildUnmerge(TmpRegs, MergeI->getOperand(Idx + 1).getReg()); + + for (unsigned j = 0; j < NumMergeRegs; ++j) + Builder.buildInstr(ConvertOp, {DstRegs[j]}, {TmpRegs[j]}); + } else { + Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg()); + } } } else if (NumMergeRegs > NumDefs) { diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index a0f21e8b19d7..fbfe71255a38 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -200,6 +200,13 @@ public: LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy); + LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, + unsigned TypeIdx, + LLT NarrowTy); + LegalizeResult fewerElementsVectorBuildVector(MachineInstr &MI, + unsigned TypeIdx, + LLT NarrowTy); + LegalizeResult reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); @@ -219,9 +226,17 @@ public: LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); + LegalizeResult lowerFMad(MachineInstr &MI); + LegalizeResult lowerUnmergeValues(MachineInstr &MI); + LegalizeResult lowerShuffleVector(MachineInstr &MI); + LegalizeResult lowerDynStackAlloc(MachineInstr &MI); + LegalizeResult lowerExtract(MachineInstr &MI); + LegalizeResult lowerInsert(MachineInstr &MI); + LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); private: MachineRegisterInfo &MRI; @@ -236,6 +251,11 @@ createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result, ArrayRef<CallLowering::ArgInfo> Args); +/// Create a libcall to memcpy et al. +LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, + MachineRegisterInfo &MRI, + MachineInstr &MI); + } // End namespace llvm. #endif diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 513c98f2d23f..1cf62d1fde59 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -331,6 +331,8 @@ class LegalizeRuleSet { /// individually handled. SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC + 2}; + SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - + MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; #endif unsigned typeIdx(unsigned TypeIdx) { @@ -342,9 +344,21 @@ class LegalizeRuleSet { #endif return TypeIdx; } - void markAllTypeIdxsAsCovered() { + + unsigned immIdx(unsigned ImmIdx) { + assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - + MCOI::OPERAND_FIRST_GENERIC_IMM) && + "Imm Index is out of bounds"); +#ifndef NDEBUG + ImmIdxsCovered.set(ImmIdx); +#endif + return ImmIdx; + } + + void markAllIdxsAsCovered() { #ifndef NDEBUG TypeIdxsCovered.set(); + ImmIdxsCovered.set(); #endif } @@ -403,6 +417,15 @@ class LegalizeRuleSet { return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), Mutation); } + /// Use the given action when type index 0 is any type in the given list and + /// imm index 0 is anything. Action should not be an action that requires + /// mutation. + LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, + std::initializer_list<LLT> Types) { + using namespace LegalityPredicates; + immIdx(0); // Inform verifier imm idx 0 is handled. + return actionIf(Action, typeInSet(typeIdx(0), Types)); + } /// Use the given action when type indexes 0 and 1 are both in the given list. /// That is, the type pair is in the cartesian product of the list. /// Action should not be an action that requires mutation. @@ -454,7 +477,7 @@ public: LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { // We have no choice but conservatively assume that the free-form // user-provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Legal, Predicate); } /// The instruction is legal when type index 0 is any type in the given list. @@ -466,6 +489,12 @@ public: LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { return actionFor(LegalizeAction::Legal, Types); } + /// The instruction is legal when type index 0 is any type in the given list + /// and imm index 0 is anything. + LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { + markAllIdxsAsCovered(); + return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); + } /// The instruction is legal when type indexes 0 and 1 along with the memory /// size and minimum alignment is any type and size tuple in the given list. LegalizeRuleSet &legalForTypesWithMemDesc( @@ -497,7 +526,7 @@ public: LegalizeRuleSet &alwaysLegal() { using namespace LegalizeMutations; - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Legal, always); } @@ -506,7 +535,7 @@ public: using namespace LegalizeMutations; // We have no choice but conservatively assume that predicate-less lowering // properly handles all type indices by design: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Lower, always); } /// The instruction is lowered if predicate is true. Keep type index 0 as the @@ -515,7 +544,7 @@ public: using namespace LegalizeMutations; // We have no choice but conservatively assume that lowering with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Lower, Predicate); } /// The instruction is lowered if predicate is true. @@ -523,7 +552,7 @@ public: LegalizeMutation Mutation) { // We have no choice but conservatively assume that lowering with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Lower, Predicate, Mutation); } /// The instruction is lowered when type index 0 is any type in the given @@ -571,7 +600,7 @@ public: LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { // We have no choice but conservatively assume that a libcall with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Libcall, Predicate); } LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { @@ -597,7 +626,7 @@ public: LegalizeMutation Mutation) { // We have no choice but conservatively assume that an action with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); } /// Narrow the scalar to the one selected by the mutation if the predicate is @@ -606,7 +635,7 @@ public: LegalizeMutation Mutation) { // We have no choice but conservatively assume that an action with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); } @@ -616,7 +645,7 @@ public: LegalizeMutation Mutation) { // We have no choice but conservatively assume that an action with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); } /// Remove elements to reach the type selected by the mutation if the @@ -625,7 +654,7 @@ public: LegalizeMutation Mutation) { // We have no choice but conservatively assume that an action with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); } @@ -640,11 +669,15 @@ public: return actionIf(LegalizeAction::Unsupported, LegalityPredicates::memSizeInBytesNotPow2(0)); } + LegalizeRuleSet &lowerIfMemSizeNotPow2() { + return actionIf(LegalizeAction::Lower, + LegalityPredicates::memSizeInBytesNotPow2(0)); + } LegalizeRuleSet &customIf(LegalityPredicate Predicate) { // We have no choice but conservatively assume that a custom action with a // free-form user provided Predicate properly handles all type indices: - markAllTypeIdxsAsCovered(); + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Custom, Predicate); } LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { @@ -882,6 +915,10 @@ public: /// LegalizeRuleSet in any way at all. /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; + /// Check if there is no imm index which is obviously not handled by the + /// LegalizeRuleSet in any way at all. + /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. + bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; /// Apply the ruleset to the given LegalityQuery. LegalizeActionStep apply(const LegalityQuery &Query) const; diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index 13eddd9539fa..be12341f5763 100644 --- a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -21,7 +21,7 @@ namespace llvm { namespace MIPatternMatch { template <typename Reg, typename Pattern> -bool mi_match(Reg R, MachineRegisterInfo &MRI, Pattern &&P) { +bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P) { return P.match(MRI, R); } @@ -30,7 +30,7 @@ template <typename SubPatternT> struct OneUse_match { SubPatternT SubPat; OneUse_match(const SubPatternT &SP) : SubPat(SP) {} - bool match(MachineRegisterInfo &MRI, unsigned Reg) { + bool match(const MachineRegisterInfo &MRI, unsigned Reg) { return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg); } }; @@ -71,7 +71,7 @@ inline operand_type_match m_Reg() { return operand_type_match(); } /// Matching combinators. template <typename... Preds> struct And { template <typename MatchSrc> - bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) { return true; } }; @@ -83,14 +83,14 @@ struct And<Pred, Preds...> : And<Preds...> { : And<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) { } template <typename MatchSrc> - bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) { return P.match(MRI, src) && And<Preds...>::match(MRI, src); } }; template <typename... Preds> struct Or { template <typename MatchSrc> - bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) { return false; } }; @@ -101,7 +101,7 @@ struct Or<Pred, Preds...> : Or<Preds...> { Or(Pred &&p, Preds &&... preds) : Or<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {} template <typename MatchSrc> - bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) { return P.match(MRI, src) || Or<Preds...>::match(MRI, src); } }; @@ -175,7 +175,8 @@ struct BinaryOp_match { RHS_P R; BinaryOp_match(const LHS_P &LHS, const RHS_P &RHS) : L(LHS), R(RHS) {} - template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) { + template <typename OpTy> + bool match(const MachineRegisterInfo &MRI, OpTy &&Op) { MachineInstr *TmpMI; if (mi_match(Op, MRI, m_MInstr(TmpMI))) { if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 3) { @@ -242,7 +243,8 @@ template <typename SrcTy, unsigned Opcode> struct UnaryOp_match { SrcTy L; UnaryOp_match(const SrcTy &LHS) : L(LHS) {} - template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) { + template <typename OpTy> + bool match(const MachineRegisterInfo &MRI, OpTy &&Op) { MachineInstr *TmpMI; if (mi_match(Op, MRI, m_MInstr(TmpMI))) { if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 2) { @@ -323,7 +325,7 @@ struct CheckType { LLT Ty; CheckType(const LLT &Ty) : Ty(Ty) {} - bool match(MachineRegisterInfo &MRI, unsigned Reg) { + bool match(const MachineRegisterInfo &MRI, unsigned Reg) { return MRI.getType(Reg) == Ty; } }; diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 10d712176b1b..416f9c19f794 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -122,14 +122,22 @@ class SrcOp { MachineInstrBuilder SrcMIB; Register Reg; CmpInst::Predicate Pred; + int64_t Imm; }; public: - enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate }; + enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate, Ty_Imm }; SrcOp(Register R) : Reg(R), Ty(SrcType::Ty_Reg) {} SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {} SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {} SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {} + /// Use of registers held in unsigned integer variables (or more rarely signed + /// integers) is no longer permitted to avoid ambiguity with upcoming support + /// for immediates. + SrcOp(unsigned) = delete; + SrcOp(int) = delete; + SrcOp(uint64_t V) : Imm(V), Ty(SrcType::Ty_Imm) {} + SrcOp(int64_t V) : Imm(V), Ty(SrcType::Ty_Imm) {} void addSrcToMIB(MachineInstrBuilder &MIB) const { switch (Ty) { @@ -142,12 +150,16 @@ public: case SrcType::Ty_MIB: MIB.addUse(SrcMIB->getOperand(0).getReg()); break; + case SrcType::Ty_Imm: + MIB.addImm(Imm); + break; } } LLT getLLTTy(const MachineRegisterInfo &MRI) const { switch (Ty) { case SrcType::Ty_Predicate: + case SrcType::Ty_Imm: llvm_unreachable("Not a register operand"); case SrcType::Ty_Reg: return MRI.getType(Reg); @@ -160,6 +172,7 @@ public: Register getReg() const { switch (Ty) { case SrcType::Ty_Predicate: + case SrcType::Ty_Imm: llvm_unreachable("Not a register operand"); case SrcType::Ty_Reg: return Reg; @@ -178,6 +191,15 @@ public: } } + int64_t getImm() const { + switch (Ty) { + case SrcType::Ty_Imm: + return Imm; + default: + llvm_unreachable("Not an immediate"); + } + } + SrcType getSrcOpKind() const { return Ty; } private: @@ -348,6 +370,17 @@ public: /// given. Convert "llvm.dbg.label Label" to "DBG_LABEL Label". MachineInstrBuilder buildDbgLabel(const MDNode *Label); + /// Build and insert \p Res = G_DYN_STACKALLOC \p Size, \p Align + /// + /// G_DYN_STACKALLOC does a dynamic stack allocation and writes the address of + /// the allocated memory into \p Res. + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, + unsigned Align); + /// Build and insert \p Res = G_FRAME_INDEX \p Idx /// /// G_FRAME_INDEX materializes the address of an alloca value or other @@ -489,11 +522,21 @@ public: return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src}); } + /// Build and insert a G_INTTOPTR instruction. + MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_INTTOPTR, {Dst}, {Src}); + } + /// Build and insert \p Dst = G_BITCAST \p Src MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src) { return buildInstr(TargetOpcode::G_BITCAST, {Dst}, {Src}); } + /// Build and insert \p Dst = G_ADDRSPACE_CAST \p Src + MachineInstrBuilder buildAddrSpaceCast(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_ADDRSPACE_CAST, {Dst}, {Src}); + } + /// \return The opcode of the extension the target wants to use for boolean /// values. unsigned getBoolExtOp(bool IsVec, bool IsFP) const; @@ -867,7 +910,8 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, - const SrcOp &Op0, const SrcOp &Op1); + const SrcOp &Op0, const SrcOp &Op1, + Optional<unsigned> Flags = None); /// Build and insert a \p Res = G_SELECT \p Tst, \p Op0, \p Op1 /// @@ -880,7 +924,8 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, - const SrcOp &Op0, const SrcOp &Op1); + const SrcOp &Op0, const SrcOp &Op1, + Optional<unsigned> Flags = None); /// Build and insert \p Res = G_INSERT_VECTOR_ELT \p Val, /// \p Elt, \p Idx @@ -961,8 +1006,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes, - Register Addr, Register Val, + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, + const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`. @@ -1135,6 +1180,16 @@ public: MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr, Register Val, MachineMemOperand &MMO); + /// Build and insert `OldValRes<def> = G_ATOMICRMW_FADD Addr, Val, MMO`. + MachineInstrBuilder buildAtomicRMWFAdd( + const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_FSUB Addr, Val, MMO`. + MachineInstrBuilder buildAtomicRMWFSub( + const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO); + /// Build and insert `G_FENCE Ordering, Scope`. MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); @@ -1210,6 +1265,12 @@ public: return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags); } + MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FMUL, {Dst}, {Src0, Src1}, Flags); + } + MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional<unsigned> Flags = None) { @@ -1300,8 +1361,9 @@ public: /// Build and insert \p Res = G_FADD \p Op0, \p Op1 MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, - const SrcOp &Src1) { - return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1}); + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_FSUB \p Op0, \p Op1 @@ -1316,14 +1378,23 @@ public: return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2}); } + /// Build and insert \p Res = G_FMAD \p Op0, \p Op1, \p Op2 + MachineInstrBuilder buildFMAD(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, const SrcOp &Src2, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FMAD, {Dst}, {Src0, Src1, Src2}, Flags); + } + /// Build and insert \p Res = G_FNEG \p Op0 - MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0) { - return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0}); + MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0}, Flags); } /// Build and insert \p Res = G_FABS \p Op0 - MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0) { - return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0}); + MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0}, Flags); } /// Build and insert \p Dst = G_FCANONICALIZE \p Src0 diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h index 4cdaa48fb689..8af2853473c2 100644 --- a/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/include/llvm/CodeGen/GlobalISel/Utils.h @@ -16,6 +16,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/Register.h" +#include "llvm/Support/LowLevelTypeImpl.h" +#include "llvm/Support/MachineValueType.h" namespace llvm { @@ -117,14 +119,16 @@ struct ValueAndVReg { unsigned VReg; }; /// If \p VReg is defined by a statically evaluable chain of -/// instructions rooted on a G_CONSTANT (\p LookThroughInstrs == true) -/// and that constant fits in int64_t, returns its value as well as -/// the virtual register defined by this G_CONSTANT. -/// When \p LookThroughInstrs == false, this function behaves like +/// instructions rooted on a G_F/CONSTANT (\p LookThroughInstrs == true) +/// and that constant fits in int64_t, returns its value as well as the +/// virtual register defined by this G_F/CONSTANT. +/// When \p LookThroughInstrs == false this function behaves like /// getConstantVRegVal. +/// When \p HandleFConstants == false the function bails on G_FCONSTANTs. Optional<ValueAndVReg> getConstantVRegValWithLookThrough(unsigned VReg, const MachineRegisterInfo &MRI, - bool LookThroughInstrs = true); + bool LookThroughInstrs = true, + bool HandleFConstants = true); const ConstantFP* getConstantFPVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); @@ -151,6 +155,9 @@ Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, const unsigned Op2, const MachineRegisterInfo &MRI); +Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const unsigned Op1, + uint64_t Imm, const MachineRegisterInfo &MRI); + /// Returns true if \p Val can be assumed to never be a NaN. If \p SNaN is true, /// this returns if \p Val can be assumed to never be a signaling NaN. bool isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, @@ -161,5 +168,10 @@ inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) { return isKnownNeverNaN(Val, MRI, true); } +/// Get a rough equivalent of an MVT for a given LLT. +MVT getMVTForLLT(LLT Ty); +/// Get a rough equivalent of an LLT for a given MVT. +LLT getLLTForMVT(MVT Ty); + } // End namespace llvm. #endif diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index acf27dcc5fab..658ad31fa2a6 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -281,7 +281,7 @@ namespace ISD { /// Same as the corresponding unsaturated fixed point instructions, but the /// result is clamped between the min and max values representable by the /// bits of the first 2 operands. - SMULFIXSAT, + SMULFIXSAT, UMULFIXSAT, /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, @@ -301,6 +301,14 @@ namespace ISD { STRICT_FEXP, STRICT_FEXP2, STRICT_FLOG, STRICT_FLOG10, STRICT_FLOG2, STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, + STRICT_LROUND, STRICT_LLROUND, STRICT_LRINT, STRICT_LLRINT, + + /// STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or + /// unsigned integer. These have the same semantics as fptosi and fptoui + /// in IR. + /// They are used to limit optimizations while the DAG is being optimized. + STRICT_FP_TO_SINT, + STRICT_FP_TO_UINT, /// X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating /// point type down to the precision of the destination VT. TRUNC is a @@ -398,6 +406,13 @@ namespace ISD { /// than the vector element type, and is implicitly truncated to it. SCALAR_TO_VECTOR, + /// SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL + /// duplicated in all lanes. The type of the operand must match the vector + /// element type, except when they are integer types. In this case the + /// operand is allowed to be wider than the vector element type, and is + /// implicitly truncated to it. + SPLAT_VECTOR, + /// MULHU/MULHS - Multiply high - Multiply two integers of type iN, /// producing an unsigned/signed value of type i[2*N], then return the top /// part. @@ -569,13 +584,6 @@ namespace ISD { /// 3 Round to -inf FLT_ROUNDS_, - /// X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and - /// rounds it to a floating point value. It then promotes it and returns it - /// in a register of the same size. This operation effectively just - /// discards excess precision. The type to round down to is specified by - /// the VT operand, a VTSDNode. - FP_ROUND_INREG, - /// X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type. FP_EXTEND, @@ -958,6 +966,23 @@ namespace ISD { static const int LAST_INDEXED_MODE = POST_DEC + 1; //===--------------------------------------------------------------------===// + /// MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's + /// index parameter when calculating addresses. + /// + /// SIGNED_SCALED Addr = Base + ((signed)Index * sizeof(element)) + /// SIGNED_UNSCALED Addr = Base + (signed)Index + /// UNSIGNED_SCALED Addr = Base + ((unsigned)Index * sizeof(element)) + /// UNSIGNED_UNSCALED Addr = Base + (unsigned)Index + enum MemIndexType { + SIGNED_SCALED = 0, + SIGNED_UNSCALED, + UNSIGNED_SCALED, + UNSIGNED_UNSCALED + }; + + static const int LAST_MEM_INDEX_TYPE = UNSIGNED_UNSCALED + 1; + + //===--------------------------------------------------------------------===// /// LoadExtType enum - This enum defines the three variants of LOADEXT /// (load with extension). /// diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index 8bb88165d3e1..290a2381d9c9 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -189,6 +189,10 @@ namespace llvm { return start == Other.start && end == Other.end; } + bool operator!=(const Segment &Other) const { + return !(*this == Other); + } + void dump() const; }; @@ -224,7 +228,7 @@ namespace llvm { /// Constructs a new LiveRange object. LiveRange(bool UseSegmentSet = false) - : segmentSet(UseSegmentSet ? llvm::make_unique<SegmentSet>() + : segmentSet(UseSegmentSet ? std::make_unique<SegmentSet>() : nullptr) {} /// Constructs a new LiveRange object by copying segments and valnos from diff --git a/include/llvm/CodeGen/LiveIntervals.h b/include/llvm/CodeGen/LiveIntervals.h index 588b0f9cf39c..888d72b87bd1 100644 --- a/include/llvm/CodeGen/LiveIntervals.h +++ b/include/llvm/CodeGen/LiveIntervals.h @@ -111,30 +111,31 @@ class VirtRegMap; const MachineBlockFrequencyInfo *MBFI, const MachineBasicBlock *MBB); - LiveInterval &getInterval(unsigned Reg) { + LiveInterval &getInterval(Register Reg) { if (hasInterval(Reg)) - return *VirtRegIntervals[Reg]; + return *VirtRegIntervals[Reg.id()]; else return createAndComputeVirtRegInterval(Reg); } - const LiveInterval &getInterval(unsigned Reg) const { + const LiveInterval &getInterval(Register Reg) const { return const_cast<LiveIntervals*>(this)->getInterval(Reg); } - bool hasInterval(unsigned Reg) const { - return VirtRegIntervals.inBounds(Reg) && VirtRegIntervals[Reg]; + bool hasInterval(Register Reg) const { + return VirtRegIntervals.inBounds(Reg.id()) && + VirtRegIntervals[Reg.id()]; } /// Interval creation. - LiveInterval &createEmptyInterval(unsigned Reg) { + LiveInterval &createEmptyInterval(Register Reg) { assert(!hasInterval(Reg) && "Interval already exists!"); - VirtRegIntervals.grow(Reg); - VirtRegIntervals[Reg] = createInterval(Reg); - return *VirtRegIntervals[Reg]; + VirtRegIntervals.grow(Reg.id()); + VirtRegIntervals[Reg.id()] = createInterval(Reg); + return *VirtRegIntervals[Reg.id()]; } - LiveInterval &createAndComputeVirtRegInterval(unsigned Reg) { + LiveInterval &createAndComputeVirtRegInterval(Register Reg) { LiveInterval &LI = createEmptyInterval(Reg); computeVirtRegInterval(LI); return LI; diff --git a/include/llvm/CodeGen/LiveRangeCalc.h b/include/llvm/CodeGen/LiveRangeCalc.h new file mode 100644 index 000000000000..08026c05733c --- /dev/null +++ b/include/llvm/CodeGen/LiveRangeCalc.h @@ -0,0 +1,295 @@ +//===- LiveRangeCalc.h - Calculate live ranges ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The LiveRangeCalc class can be used to compute live ranges from scratch. It +// caches information about values in the CFG to speed up repeated operations +// on the same live range. The cache can be shared by non-overlapping live +// ranges. SplitKit uses that when computing the live range of split products. +// +// A low-level interface is available to clients that know where a variable is +// live, but don't know which value it has as every point. LiveRangeCalc will +// propagate values down the dominator tree, and even insert PHI-defs where +// needed. SplitKit uses this faster interface when possible. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_LIVERANGECALC_H +#define LLVM_LIB_CODEGEN_LIVERANGECALC_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/LiveInterval.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/MC/LaneBitmask.h" +#include <utility> + +namespace llvm { + +template <class NodeT> class DomTreeNodeBase; +class MachineDominatorTree; +class MachineFunction; +class MachineRegisterInfo; + +using MachineDomTreeNode = DomTreeNodeBase<MachineBasicBlock>; + +class LiveRangeCalc { + const MachineFunction *MF = nullptr; + const MachineRegisterInfo *MRI = nullptr; + SlotIndexes *Indexes = nullptr; + MachineDominatorTree *DomTree = nullptr; + VNInfo::Allocator *Alloc = nullptr; + + /// LiveOutPair - A value and the block that defined it. The domtree node is + /// redundant, it can be computed as: MDT[Indexes.getMBBFromIndex(VNI->def)]. + using LiveOutPair = std::pair<VNInfo *, MachineDomTreeNode *>; + + /// LiveOutMap - Map basic blocks to the value leaving the block. + using LiveOutMap = IndexedMap<LiveOutPair, MBB2NumberFunctor>; + + /// Bit vector of active entries in LiveOut, also used as a visited set by + /// findReachingDefs. One entry per basic block, indexed by block number. + /// This is kept as a separate bit vector because it can be cleared quickly + /// when switching live ranges. + BitVector Seen; + + /// Map LiveRange to sets of blocks (represented by bit vectors) that + /// in the live range are defined on entry and undefined on entry. + /// A block is defined on entry if there is a path from at least one of + /// the defs in the live range to the entry of the block, and conversely, + /// a block is undefined on entry, if there is no such path (i.e. no + /// definition reaches the entry of the block). A single LiveRangeCalc + /// object is used to track live-out information for multiple registers + /// in live range splitting (which is ok, since the live ranges of these + /// registers do not overlap), but the defined/undefined information must + /// be kept separate for each individual range. + /// By convention, EntryInfoMap[&LR] = { Defined, Undefined }. + using EntryInfoMap = DenseMap<LiveRange *, std::pair<BitVector, BitVector>>; + EntryInfoMap EntryInfos; + + /// Map each basic block where a live range is live out to the live-out value + /// and its defining block. + /// + /// For every basic block, MBB, one of these conditions shall be true: + /// + /// 1. !Seen.count(MBB->getNumber()) + /// Blocks without a Seen bit are ignored. + /// 2. LiveOut[MBB].second.getNode() == MBB + /// The live-out value is defined in MBB. + /// 3. forall P in preds(MBB): LiveOut[P] == LiveOut[MBB] + /// The live-out value passses through MBB. All predecessors must carry + /// the same value. + /// + /// The domtree node may be null, it can be computed. + /// + /// The map can be shared by multiple live ranges as long as no two are + /// live-out of the same block. + LiveOutMap Map; + + /// LiveInBlock - Information about a basic block where a live range is known + /// to be live-in, but the value has not yet been determined. + struct LiveInBlock { + // The live range set that is live-in to this block. The algorithms can + // handle multiple non-overlapping live ranges simultaneously. + LiveRange &LR; + + // DomNode - Dominator tree node for the block. + // Cleared when the final value has been determined and LI has been updated. + MachineDomTreeNode *DomNode; + + // Position in block where the live-in range ends, or SlotIndex() if the + // range passes through the block. When the final value has been + // determined, the range from the block start to Kill will be added to LI. + SlotIndex Kill; + + // Live-in value filled in by updateSSA once it is known. + VNInfo *Value = nullptr; + + LiveInBlock(LiveRange &LR, MachineDomTreeNode *node, SlotIndex kill) + : LR(LR), DomNode(node), Kill(kill) {} + }; + + /// LiveIn - Work list of blocks where the live-in value has yet to be + /// determined. This list is typically computed by findReachingDefs() and + /// used as a work list by updateSSA(). The low-level interface may also be + /// used to add entries directly. + SmallVector<LiveInBlock, 16> LiveIn; + + /// Check if the entry to block @p MBB can be reached by any of the defs + /// in @p LR. Return true if none of the defs reach the entry to @p MBB. + bool isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs, + MachineBasicBlock &MBB, BitVector &DefOnEntry, + BitVector &UndefOnEntry); + + /// Find the set of defs that can reach @p Kill. @p Kill must belong to + /// @p UseMBB. + /// + /// If exactly one def can reach @p UseMBB, and the def dominates @p Kill, + /// all paths from the def to @p UseMBB are added to @p LR, and the function + /// returns true. + /// + /// If multiple values can reach @p UseMBB, the blocks that need @p LR to be + /// live in are added to the LiveIn array, and the function returns false. + /// + /// The array @p Undef provides the locations where the range @p LR becomes + /// undefined by <def,read-undef> operands on other subranges. If @p Undef + /// is non-empty and @p Kill is jointly dominated only by the entries of + /// @p Undef, the function returns false. + /// + /// PhysReg, when set, is used to verify live-in lists on basic blocks. + bool findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, SlotIndex Use, + unsigned PhysReg, ArrayRef<SlotIndex> Undefs); + + /// updateSSA - Compute the values that will be live in to all requested + /// blocks in LiveIn. Create PHI-def values as required to preserve SSA form. + /// + /// Every live-in block must be jointly dominated by the added live-out + /// blocks. No values are read from the live ranges. + void updateSSA(); + + /// Transfer information from the LiveIn vector to the live ranges and update + /// the given @p LiveOuts. + void updateFromLiveIns(); + + /// Extend the live range of @p LR to reach all uses of Reg. + /// + /// If @p LR is a main range, or if @p LI is null, then all uses must be + /// jointly dominated by the definitions from @p LR. If @p LR is a subrange + /// of the live interval @p LI, corresponding to lane mask @p LaneMask, + /// all uses must be jointly dominated by the definitions from @p LR + /// together with definitions of other lanes where @p LR becomes undefined + /// (via <def,read-undef> operands). + /// If @p LR is a main range, the @p LaneMask should be set to ~0, i.e. + /// LaneBitmask::getAll(). + void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask, + LiveInterval *LI = nullptr); + + /// Reset Map and Seen fields. + void resetLiveOutMap(); + +public: + LiveRangeCalc() = default; + + //===--------------------------------------------------------------------===// + // High-level interface. + //===--------------------------------------------------------------------===// + // + // Calculate live ranges from scratch. + // + + /// reset - Prepare caches for a new set of non-overlapping live ranges. The + /// caches must be reset before attempting calculations with a live range + /// that may overlap a previously computed live range, and before the first + /// live range in a function. If live ranges are not known to be + /// non-overlapping, call reset before each. + void reset(const MachineFunction *mf, SlotIndexes *SI, + MachineDominatorTree *MDT, VNInfo::Allocator *VNIA); + + //===--------------------------------------------------------------------===// + // Mid-level interface. + //===--------------------------------------------------------------------===// + // + // Modify existing live ranges. + // + + /// Extend the live range of @p LR to reach @p Use. + /// + /// The existing values in @p LR must be live so they jointly dominate @p Use. + /// If @p Use is not dominated by a single existing value, PHI-defs are + /// inserted as required to preserve SSA form. + /// + /// PhysReg, when set, is used to verify live-in lists on basic blocks. + void extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg, + ArrayRef<SlotIndex> Undefs); + + /// createDeadDefs - Create a dead def in LI for every def operand of Reg. + /// Each instruction defining Reg gets a new VNInfo with a corresponding + /// minimal live range. + void createDeadDefs(LiveRange &LR, unsigned Reg); + + /// Extend the live range of @p LR to reach all uses of Reg. + /// + /// All uses must be jointly dominated by existing liveness. PHI-defs are + /// inserted as needed to preserve SSA form. + void extendToUses(LiveRange &LR, unsigned PhysReg) { + extendToUses(LR, PhysReg, LaneBitmask::getAll()); + } + + /// Calculates liveness for the register specified in live interval @p LI. + /// Creates subregister live ranges as needed if subreg liveness tracking is + /// enabled. + void calculate(LiveInterval &LI, bool TrackSubRegs); + + /// For live interval \p LI with correct SubRanges construct matching + /// information for the main live range. Expects the main live range to not + /// have any segments or value numbers. + void constructMainRangeFromSubranges(LiveInterval &LI); + + //===--------------------------------------------------------------------===// + // Low-level interface. + //===--------------------------------------------------------------------===// + // + // These functions can be used to compute live ranges where the live-in and + // live-out blocks are already known, but the SSA value in each block is + // unknown. + // + // After calling reset(), add known live-out values and known live-in blocks. + // Then call calculateValues() to compute the actual value that is + // live-in to each block, and add liveness to the live ranges. + // + + /// setLiveOutValue - Indicate that VNI is live out from MBB. The + /// calculateValues() function will not add liveness for MBB, the caller + /// should take care of that. + /// + /// VNI may be null only if MBB is a live-through block also passed to + /// addLiveInBlock(). + void setLiveOutValue(MachineBasicBlock *MBB, VNInfo *VNI) { + Seen.set(MBB->getNumber()); + Map[MBB] = LiveOutPair(VNI, nullptr); + } + + /// addLiveInBlock - Add a block with an unknown live-in value. This + /// function can only be called once per basic block. Once the live-in value + /// has been determined, calculateValues() will add liveness to LI. + /// + /// @param LR The live range that is live-in to the block. + /// @param DomNode The domtree node for the block. + /// @param Kill Index in block where LI is killed. If the value is + /// live-through, set Kill = SLotIndex() and also call + /// setLiveOutValue(MBB, 0). + void addLiveInBlock(LiveRange &LR, MachineDomTreeNode *DomNode, + SlotIndex Kill = SlotIndex()) { + LiveIn.push_back(LiveInBlock(LR, DomNode, Kill)); + } + + /// calculateValues - Calculate the value that will be live-in to each block + /// added with addLiveInBlock. Add PHI-def values as needed to preserve SSA + /// form. Add liveness to all live-in blocks up to the Kill point, or the + /// whole block for live-through blocks. + /// + /// Every predecessor of a live-in block must have been given a value with + /// setLiveOutValue, the value may be null for live-trough blocks. + void calculateValues(); + + /// A diagnostic function to check if the end of the block @p MBB is + /// jointly dominated by the blocks corresponding to the slot indices + /// in @p Defs. This function is mainly for use in self-verification + /// checks. + LLVM_ATTRIBUTE_UNUSED + static bool isJointlyDominated(const MachineBasicBlock *MBB, + ArrayRef<SlotIndex> Defs, + const SlotIndexes &Indexes); +}; + +} // end namespace llvm + +#endif // LLVM_LIB_CODEGEN_LIVERANGECALC_H diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h index 7dbb2feab8bf..314afad92970 100644 --- a/include/llvm/CodeGen/LiveRegUnits.h +++ b/include/llvm/CodeGen/LiveRegUnits.h @@ -53,8 +53,8 @@ public: ModifiedRegUnits.addRegsInMask(O->getRegMask()); if (!O->isReg()) continue; - unsigned Reg = O->getReg(); - if (!TargetRegisterInfo::isPhysicalRegister(Reg)) + Register Reg = O->getReg(); + if (!Reg.isPhysical()) continue; if (O->isDef()) { // Some architectures (e.g. AArch64 XZR/WZR) have registers that are diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h index 94e76a75e8da..069d0aa45095 100644 --- a/include/llvm/CodeGen/MIRYamlMapping.h +++ b/include/llvm/CodeGen/MIRYamlMapping.h @@ -314,6 +314,7 @@ struct ScalarEnumerationTraits<TargetStackID::Value> { static void enumeration(yaml::IO &IO, TargetStackID::Value &ID) { IO.enumCase(ID, "default", TargetStackID::Default); IO.enumCase(ID, "sgpr-spill", TargetStackID::SGPRSpill); + IO.enumCase(ID, "sve-vec", TargetStackID::SVEVector); IO.enumCase(ID, "noalloc", TargetStackID::NoAlloc); } }; diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index 333d0a78618c..ccdde78a0b22 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -103,9 +103,9 @@ private: using LiveInVector = std::vector<RegisterMaskPair>; LiveInVector LiveIns; - /// Alignment of the basic block. Zero if the basic block does not need to be - /// aligned. The alignment is specified as log2(bytes). - unsigned Alignment = 0; + /// Alignment of the basic block. One if the basic block does not need to be + /// aligned. + Align Alignment; /// Indicate that this basic block is entered via an exception handler. bool IsEHPad = false; @@ -312,7 +312,7 @@ public: /// Adds the specified register as a live in. Note that it is an error to add /// the same register to the same set more than once unless the intention is /// to call sortUniqueLiveIns after all registers are added. - void addLiveIn(MCPhysReg PhysReg, + void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask = LaneBitmask::getAll()) { LiveIns.push_back(RegisterMaskPair(PhysReg, LaneMask)); } @@ -331,7 +331,7 @@ public: /// Add PhysReg as live in to this block, and ensure that there is a copy of /// PhysReg to a virtual register of class RC. Return the virtual register /// that is a copy of the live in PhysReg. - unsigned addLiveIn(MCPhysReg PhysReg, const TargetRegisterClass *RC); + unsigned addLiveIn(MCRegister PhysReg, const TargetRegisterClass *RC); /// Remove the specified register from the live in set. void removeLiveIn(MCPhysReg Reg, @@ -372,13 +372,11 @@ public: /// \see getBeginClobberMask() const uint32_t *getEndClobberMask(const TargetRegisterInfo *TRI) const; - /// Return alignment of the basic block. The alignment is specified as - /// log2(bytes). - unsigned getAlignment() const { return Alignment; } + /// Return alignment of the basic block. + Align getAlignment() const { return Alignment; } - /// Set alignment of the basic block. The alignment is specified as - /// log2(bytes). - void setAlignment(unsigned Align) { Alignment = Align; } + /// Set alignment of the basic block. + void setAlignment(Align A) { Alignment = A; } /// Returns true if the block is a landing pad. That is this basic block is /// entered via an exception handler. @@ -636,6 +634,18 @@ public: return Insts.insertAfter(I.getInstrIterator(), MI); } + /// If I is bundled then insert MI into the instruction list after the end of + /// the bundle, otherwise insert MI immediately after I. + instr_iterator insertAfterBundle(instr_iterator I, MachineInstr *MI) { + assert((I == instr_end() || I->getParent() == this) && + "iterator points outside of basic block"); + assert(!MI->isBundledWithPred() && !MI->isBundledWithSucc() && + "Cannot insert instruction with bundle flags"); + while (I->isBundledWithSucc()) + ++I; + return Insts.insertAfter(I, MI); + } + /// Remove an instruction from the instruction list and delete it. /// /// If the instruction is part of a bundle, the other instructions in the @@ -723,6 +733,10 @@ public: /// CFG so that it branches to 'New' instead. void ReplaceUsesOfBlockWith(MachineBasicBlock *Old, MachineBasicBlock *New); + /// Update all phi nodes in this basic block to refer to basic block \p New + /// instead of basic block \p Old. + void replacePhiUsesWith(MachineBasicBlock *Old, MachineBasicBlock *New); + /// Various pieces of code can cause excess edges in the CFG to be inserted. /// If we have proven that MBB can only branch to DestA and DestB, remove any /// other MBB successors from the CFG. DestA and DestB can be null. Besides diff --git a/include/llvm/CodeGen/MachineCombinerPattern.h b/include/llvm/CodeGen/MachineCombinerPattern.h index 4f4034baf801..503227222207 100644 --- a/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/include/llvm/CodeGen/MachineCombinerPattern.h @@ -39,6 +39,10 @@ enum class MachineCombinerPattern { MULADDXI_OP1, MULSUBXI_OP1, // Floating Point + FMULADDH_OP1, + FMULADDH_OP2, + FMULSUBH_OP1, + FMULSUBH_OP2, FMULADDS_OP1, FMULADDS_OP2, FMULSUBS_OP1, @@ -47,16 +51,25 @@ enum class MachineCombinerPattern { FMULADDD_OP2, FMULSUBD_OP1, FMULSUBD_OP2, + FNMULSUBH_OP1, FNMULSUBS_OP1, FNMULSUBD_OP1, FMLAv1i32_indexed_OP1, FMLAv1i32_indexed_OP2, FMLAv1i64_indexed_OP1, FMLAv1i64_indexed_OP2, + FMLAv4f16_OP1, + FMLAv4f16_OP2, + FMLAv8f16_OP1, + FMLAv8f16_OP2, FMLAv2f32_OP2, FMLAv2f32_OP1, FMLAv2f64_OP1, FMLAv2f64_OP2, + FMLAv4i16_indexed_OP1, + FMLAv4i16_indexed_OP2, + FMLAv8i16_indexed_OP1, + FMLAv8i16_indexed_OP2, FMLAv2i32_indexed_OP1, FMLAv2i32_indexed_OP2, FMLAv2i64_indexed_OP1, @@ -67,10 +80,18 @@ enum class MachineCombinerPattern { FMLAv4i32_indexed_OP2, FMLSv1i32_indexed_OP2, FMLSv1i64_indexed_OP2, + FMLSv4f16_OP1, + FMLSv4f16_OP2, + FMLSv8f16_OP1, + FMLSv8f16_OP2, FMLSv2f32_OP1, FMLSv2f32_OP2, FMLSv2f64_OP1, FMLSv2f64_OP2, + FMLSv4i16_indexed_OP1, + FMLSv4i16_indexed_OP2, + FMLSv8i16_indexed_OP1, + FMLSv8i16_indexed_OP2, FMLSv2i32_indexed_OP1, FMLSv2i32_indexed_OP2, FMLSv2i64_indexed_OP1, diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h index d2200080b897..e4d7a02f8c48 100644 --- a/include/llvm/CodeGen/MachineDominators.h +++ b/include/llvm/CodeGen/MachineDominators.h @@ -44,6 +44,8 @@ using MachineDomTreeNode = DomTreeNodeBase<MachineBasicBlock>; /// compute a normal dominator tree. /// class MachineDominatorTree : public MachineFunctionPass { + using DomTreeT = DomTreeBase<MachineBasicBlock>; + /// Helper structure used to hold all the basic blocks /// involved in the split of a critical edge. struct CriticalEdge { @@ -65,8 +67,8 @@ class MachineDominatorTree : public MachineFunctionPass { /// such as BB == elt.NewBB. mutable SmallSet<MachineBasicBlock *, 32> NewBBs; - /// The DominatorTreeBase that is used to compute a normal dominator tree - std::unique_ptr<DomTreeBase<MachineBasicBlock>> DT; + /// The DominatorTreeBase that is used to compute a normal dominator tree. + std::unique_ptr<DomTreeT> DT; /// Apply all the recorded critical edges to the DT. /// This updates the underlying DT information in a way that uses @@ -80,8 +82,8 @@ public: MachineDominatorTree(); - DomTreeBase<MachineBasicBlock> &getBase() { - if (!DT) DT.reset(new DomTreeBase<MachineBasicBlock>()); + DomTreeT &getBase() { + if (!DT) DT.reset(new DomTreeT()); applySplitCriticalEdges(); return *DT; } @@ -92,31 +94,30 @@ public: /// multiple blocks if we are computing post dominators. For forward /// dominators, this will always be a single block (the entry node). /// - inline const SmallVectorImpl<MachineBasicBlock*> &getRoots() const { + const SmallVectorImpl<MachineBasicBlock*> &getRoots() const { applySplitCriticalEdges(); return DT->getRoots(); } - inline MachineBasicBlock *getRoot() const { + MachineBasicBlock *getRoot() const { applySplitCriticalEdges(); return DT->getRoot(); } - inline MachineDomTreeNode *getRootNode() const { + MachineDomTreeNode *getRootNode() const { applySplitCriticalEdges(); return DT->getRootNode(); } bool runOnMachineFunction(MachineFunction &F) override; - inline bool dominates(const MachineDomTreeNode* A, - const MachineDomTreeNode* B) const { + bool dominates(const MachineDomTreeNode *A, + const MachineDomTreeNode *B) const { applySplitCriticalEdges(); return DT->dominates(A, B); } - inline bool dominates(const MachineBasicBlock* A, - const MachineBasicBlock* B) const { + bool dominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const { applySplitCriticalEdges(); return DT->dominates(A, B); } @@ -133,36 +134,30 @@ public: for (; &*I != A && &*I != B; ++I) /*empty*/ ; - //if(!DT.IsPostDominators) { - // A dominates B if it is found first in the basic block. - return &*I == A; - //} else { - // // A post-dominates B if B is found first in the basic block. - // return &*I == B; - //} + return &*I == A; } - inline bool properlyDominates(const MachineDomTreeNode* A, - const MachineDomTreeNode* B) const { + bool properlyDominates(const MachineDomTreeNode *A, + const MachineDomTreeNode *B) const { applySplitCriticalEdges(); return DT->properlyDominates(A, B); } - inline bool properlyDominates(const MachineBasicBlock* A, - const MachineBasicBlock* B) const { + bool properlyDominates(const MachineBasicBlock *A, + const MachineBasicBlock *B) const { applySplitCriticalEdges(); return DT->properlyDominates(A, B); } /// findNearestCommonDominator - Find nearest common dominator basic block /// for basic block A and B. If there is no such block then return NULL. - inline MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A, - MachineBasicBlock *B) { + MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A, + MachineBasicBlock *B) { applySplitCriticalEdges(); return DT->findNearestCommonDominator(A, B); } - inline MachineDomTreeNode *operator[](MachineBasicBlock *BB) const { + MachineDomTreeNode *operator[](MachineBasicBlock *BB) const { applySplitCriticalEdges(); return DT->getNode(BB); } @@ -170,7 +165,7 @@ public: /// getNode - return the (Post)DominatorTree node for the specified basic /// block. This is the same as using operator[] on this class. /// - inline MachineDomTreeNode *getNode(MachineBasicBlock *BB) const { + MachineDomTreeNode *getNode(MachineBasicBlock *BB) const { applySplitCriticalEdges(); return DT->getNode(BB); } @@ -178,8 +173,8 @@ public: /// addNewBlock - Add a new node to the dominator tree information. This /// creates a new node as a child of DomBB dominator node,linking it into /// the children list of the immediate dominator. - inline MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB, - MachineBasicBlock *DomBB) { + MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB, + MachineBasicBlock *DomBB) { applySplitCriticalEdges(); return DT->addNewBlock(BB, DomBB); } @@ -187,14 +182,14 @@ public: /// changeImmediateDominator - This method is used to update the dominator /// tree information when a node's immediate dominator changes. /// - inline void changeImmediateDominator(MachineBasicBlock *N, - MachineBasicBlock* NewIDom) { + void changeImmediateDominator(MachineBasicBlock *N, + MachineBasicBlock *NewIDom) { applySplitCriticalEdges(); DT->changeImmediateDominator(N, NewIDom); } - inline void changeImmediateDominator(MachineDomTreeNode *N, - MachineDomTreeNode* NewIDom) { + void changeImmediateDominator(MachineDomTreeNode *N, + MachineDomTreeNode *NewIDom) { applySplitCriticalEdges(); DT->changeImmediateDominator(N, NewIDom); } @@ -202,14 +197,14 @@ public: /// eraseNode - Removes a node from the dominator tree. Block must not /// dominate any other blocks. Removes node from its immediate dominator's /// children list. Deletes dominator node associated with basic block BB. - inline void eraseNode(MachineBasicBlock *BB) { + void eraseNode(MachineBasicBlock *BB) { applySplitCriticalEdges(); DT->eraseNode(BB); } /// splitBlock - BB is split and now it has one successor. Update dominator /// tree to reflect this change. - inline void splitBlock(MachineBasicBlock* NewBB) { + void splitBlock(MachineBasicBlock* NewBB) { applySplitCriticalEdges(); DT->splitBlock(NewBB); } diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index 761735120a64..01fc50d14a7f 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -14,6 +14,7 @@ #define LLVM_CODEGEN_MACHINEFRAMEINFO_H #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/DataTypes.h" #include <cassert> #include <vector> @@ -129,7 +130,7 @@ private: uint64_t Size; // The required alignment of this stack slot. - unsigned Alignment; + Align Alignment; // If true, the value of the stack object is set before // entering the function and is not modified inside the function. By @@ -180,17 +181,16 @@ private: uint8_t SSPLayout; - StackObject(uint64_t Size, unsigned Alignment, int64_t SPOffset, + StackObject(uint64_t Size, Align Alignment, int64_t SPOffset, bool IsImmutable, bool IsSpillSlot, const AllocaInst *Alloca, bool IsAliased, uint8_t StackID = 0) - : SPOffset(SPOffset), Size(Size), Alignment(Alignment), - isImmutable(IsImmutable), isSpillSlot(IsSpillSlot), - StackID(StackID), Alloca(Alloca), isAliased(IsAliased), - SSPLayout(SSPLK_None) {} + : SPOffset(SPOffset), Size(Size), Alignment(Alignment), + isImmutable(IsImmutable), isSpillSlot(IsSpillSlot), StackID(StackID), + Alloca(Alloca), isAliased(IsAliased), SSPLayout(SSPLK_None) {} }; /// The alignment of the stack. - unsigned StackAlignment; + Align StackAlignment; /// Can the stack be realigned. This can be false if the target does not /// support stack realignment, or if the user asks us not to realign the @@ -260,7 +260,7 @@ private: /// native alignment maintained by the compiler, dynamic alignment code will /// be needed. /// - unsigned MaxAlignment = 0; + Align MaxAlignment; /// Set to true if this function adjusts the stack -- e.g., /// when calling another function. This is only valid during and after @@ -304,7 +304,7 @@ private: /// Required alignment of the local object blob, which is the strictest /// alignment of any object in it. - unsigned LocalFrameMaxAlign = 0; + Align LocalFrameMaxAlign; /// Whether the local object blob needs to be allocated together. If not, /// PEI should ignore the isPreAllocated flags on the stack objects and @@ -338,8 +338,8 @@ private: public: explicit MachineFrameInfo(unsigned StackAlignment, bool StackRealignable, bool ForcedRealign) - : StackAlignment(StackAlignment), StackRealignable(StackRealignable), - ForcedRealign(ForcedRealign) {} + : StackAlignment(assumeAligned(StackAlignment)), + StackRealignable(StackRealignable), ForcedRealign(ForcedRealign) {} /// Return true if there are any stack objects in this function. bool hasStackObjects() const { return !Objects.empty(); } @@ -419,10 +419,12 @@ public: /// Required alignment of the local object blob, /// which is the strictest alignment of any object in it. - void setLocalFrameMaxAlign(unsigned Align) { LocalFrameMaxAlign = Align; } + void setLocalFrameMaxAlign(Align Alignment) { + LocalFrameMaxAlign = Alignment; + } /// Return the required alignment of the local object blob. - unsigned getLocalFrameMaxAlign() const { return LocalFrameMaxAlign; } + Align getLocalFrameMaxAlign() const { return LocalFrameMaxAlign; } /// Get whether the local allocation blob should be allocated together or /// let PEI allocate the locals in it directly. @@ -462,14 +464,14 @@ public: unsigned getObjectAlignment(int ObjectIdx) const { assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); - return Objects[ObjectIdx+NumFixedObjects].Alignment; + return Objects[ObjectIdx + NumFixedObjects].Alignment.value(); } /// setObjectAlignment - Change the alignment of the specified stack object. void setObjectAlignment(int ObjectIdx, unsigned Align) { assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); - Objects[ObjectIdx+NumFixedObjects].Alignment = Align; + Objects[ObjectIdx + NumFixedObjects].Alignment = assumeAligned(Align); // Only ensure max alignment for the default stack. if (getStackID(ObjectIdx) == 0) @@ -561,10 +563,14 @@ public: /// Return the alignment in bytes that this function must be aligned to, /// which is greater than the default stack alignment provided by the target. - unsigned getMaxAlignment() const { return MaxAlignment; } + unsigned getMaxAlignment() const { return MaxAlignment.value(); } /// Make sure the function is at least Align bytes aligned. - void ensureMaxAlignment(unsigned Align); + void ensureMaxAlignment(Align Alignment); + /// FIXME: Remove this once transition to Align is over. + inline void ensureMaxAlignment(unsigned Align) { + ensureMaxAlignment(assumeAligned(Align)); + } /// Return true if this function adjusts the stack -- e.g., /// when calling another function. This is only valid during and after @@ -728,12 +734,24 @@ public: /// Create a new statically sized stack object, returning /// a nonnegative identifier to represent it. - int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, + int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca = nullptr, uint8_t ID = 0); + /// FIXME: Remove this function when transition to Align is over. + inline int CreateStackObject(uint64_t Size, unsigned Alignment, + bool isSpillSlot, + const AllocaInst *Alloca = nullptr, + uint8_t ID = 0) { + return CreateStackObject(Size, assumeAligned(Alignment), isSpillSlot, + Alloca, ID); + } /// Create a new statically sized stack object that represents a spill slot, /// returning a nonnegative identifier to represent it. - int CreateSpillStackObject(uint64_t Size, unsigned Alignment); + int CreateSpillStackObject(uint64_t Size, Align Alignment); + /// FIXME: Remove this function when transition to Align is over. + inline int CreateSpillStackObject(uint64_t Size, unsigned Alignment) { + return CreateSpillStackObject(Size, assumeAligned(Alignment)); + } /// Remove or mark dead a statically sized stack object. void RemoveStackObject(int ObjectIdx) { @@ -744,7 +762,11 @@ public: /// Notify the MachineFrameInfo object that a variable sized object has been /// created. This must be created whenever a variable sized object is /// created, whether or not the index returned is actually used. - int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca); + int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca); + /// FIXME: Remove this function when transition to Align is over. + int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca) { + return CreateVariableSizedObject(assumeAligned(Alignment), Alloca); + } /// Returns a reference to call saved info vector for the current function. const std::vector<CalleeSavedInfo> &getCalleeSavedInfo() const { diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index 201c126ee52e..3a3176e51c51 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -36,6 +36,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Recycler.h" +#include "llvm/Target/TargetMachine.h" #include <cassert> #include <cstdint> #include <memory> @@ -277,7 +278,7 @@ class MachineFunction { unsigned FunctionNumber; /// Alignment - The alignment of the function. - unsigned Alignment; + Align Alignment; /// ExposesReturnsTwice - True if the function calls setjmp or related /// functions with attribute "returns twice", but doesn't have @@ -322,7 +323,7 @@ class MachineFunction { std::vector<std::pair<MCSymbol *, MDNode *>> CodeViewAnnotations; /// CodeView heapallocsites. - std::vector<std::tuple<MCSymbol*, MCSymbol*, DIType*>> + std::vector<std::tuple<MCSymbol *, MCSymbol *, const DIType *>> CodeViewHeapAllocSites; bool CallsEHReturn = false; @@ -400,6 +401,17 @@ private: /// Map a call instruction to call site arguments forwarding info. CallSiteInfoMap CallSitesInfo; + /// A helper function that returns call site info for a give call + /// instruction if debug entry value support is enabled. + CallSiteInfoMap::iterator getCallSiteInfo(const MachineInstr *MI) { + assert(MI->isCall() && + "Call site info refers only to call instructions!"); + + if (!Target.Options.EnableDebugEntryValues) + return CallSitesInfo.end(); + return CallSitesInfo.find(MI); + } + // Callbacks for insertion and removal. void handleInsertion(MachineInstr &MI); void handleRemoval(MachineInstr &MI); @@ -508,15 +520,16 @@ public: const WinEHFuncInfo *getWinEHFuncInfo() const { return WinEHInfo; } WinEHFuncInfo *getWinEHFuncInfo() { return WinEHInfo; } - /// getAlignment - Return the alignment (log2, not bytes) of the function. - unsigned getAlignment() const { return Alignment; } + /// getAlignment - Return the alignment of the function. + Align getAlignment() const { return Alignment; } - /// setAlignment - Set the alignment (log2, not bytes) of the function. - void setAlignment(unsigned A) { Alignment = A; } + /// setAlignment - Set the alignment of the function. + void setAlignment(Align A) { Alignment = A; } - /// ensureAlignment - Make sure the function is at least 1 << A bytes aligned. - void ensureAlignment(unsigned A) { - if (Alignment < A) Alignment = A; + /// ensureAlignment - Make sure the function is at least A bytes aligned. + void ensureAlignment(Align A) { + if (Alignment < A) + Alignment = A; } /// exposesReturnsTwice - Returns true if the function calls setjmp or @@ -935,10 +948,10 @@ public: } /// Record heapallocsites - void addCodeViewHeapAllocSite(MachineInstr *I, MDNode *MD); + void addCodeViewHeapAllocSite(MachineInstr *I, const MDNode *MD); - ArrayRef<std::tuple<MCSymbol*, MCSymbol*, DIType*>> - getCodeViewHeapAllocSites() const { + ArrayRef<std::tuple<MCSymbol *, MCSymbol *, const DIType *>> + getCodeViewHeapAllocSites() const { return CodeViewHeapAllocSites; } @@ -976,12 +989,24 @@ public: return CallSitesInfo; } - /// Update call sites info by deleting entry for \p Old call instruction. - /// If \p New is present then transfer \p Old call info to it. This function - /// should be called before removing call instruction or before replacing - /// call instruction with new one. - void updateCallSiteInfo(const MachineInstr *Old, - const MachineInstr *New = nullptr); + /// Following functions update call site info. They should be called before + /// removing, replacing or copying call instruction. + + /// Move the call site info from \p Old to \New call site info. This function + /// is used when we are replacing one call instruction with another one to + /// the same callee. + void moveCallSiteInfo(const MachineInstr *Old, + const MachineInstr *New); + + /// Erase the call site info for \p MI. It is used to remove a call + /// instruction from the instruction stream. + void eraseCallSiteInfo(const MachineInstr *MI); + + /// Copy the call site info from \p Old to \ New. Its usage is when we are + /// making a copy of the instruction that will be inserted at different point + /// of the instruction stream. + void copyCallSiteInfo(const MachineInstr *Old, + const MachineInstr *New); }; //===--------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index c82c5b137507..c94ad292ec96 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -20,11 +20,9 @@ #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" -#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" @@ -38,6 +36,7 @@ namespace llvm { +class AAResults; template <typename T> class ArrayRef; class DIExpression; class DILocalVariable; @@ -427,6 +426,22 @@ public: return getNumExplicitDefs() + MCID->getNumImplicitDefs(); } + /// Returns true if the instruction has implicit definition. + bool hasImplicitDef() const { + for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); + I != E; ++I) { + const MachineOperand &MO = getOperand(I); + if (MO.isDef() && MO.isImplicit()) + return true; + } + return false; + } + + /// Returns the implicit operands number. + unsigned getNumImplicitOperands() const { + return getNumOperands() - getNumExplicitOperands(); + } + /// Return true if operand \p OpIdx is a subregister index. bool isOperandSubregIdx(unsigned OpIdx) const { assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && @@ -602,6 +617,12 @@ public: return hasPropertyInBundle(1ULL << MCFlag, Type); } + /// Return true if this is an instruction that should go through the usual + /// legalization steps. + bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { + return hasProperty(MCID::PreISelOpcode, Type); + } + /// Return true if this instruction can have a variable number of operands. /// In this case, the variable operands will be after the normal /// operands but before the implicit definitions and uses (if any are @@ -1020,15 +1041,13 @@ public: } /// A DBG_VALUE is an entry value iff its debug expression contains the - /// DW_OP_entry_value DWARF operation. - bool isDebugEntryValue() const { - return isDebugValue() && getDebugExpression()->isEntryValue(); - } + /// DW_OP_LLVM_entry_value operation. + bool isDebugEntryValue() const; /// Return true if the instruction is a debug value which describes a part of /// a variable as unavailable. bool isUndefDebugValue() const { - return isDebugValue() && getOperand(0).isReg() && !getOperand(0).getReg(); + return isDebugValue() && getOperand(0).isReg() && !getOperand(0).getReg().isValid(); } bool isPHI() const { @@ -1140,7 +1159,7 @@ public: /// is a read of a super-register. /// This does not count partial redefines of virtual registers as reads: /// %reg1024:6 = OP. - bool readsRegister(unsigned Reg, + bool readsRegister(Register Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterUseOperandIdx(Reg, false, TRI) != -1; } @@ -1148,20 +1167,20 @@ public: /// Return true if the MachineInstr reads the specified virtual register. /// Take into account that a partial define is a /// read-modify-write operation. - bool readsVirtualRegister(unsigned Reg) const { + bool readsVirtualRegister(Register Reg) const { return readsWritesVirtualRegister(Reg).first; } /// Return a pair of bools (reads, writes) indicating if this instruction /// reads or writes Reg. This also considers partial defines. /// If Ops is not null, all operand indices for Reg are added. - std::pair<bool,bool> readsWritesVirtualRegister(unsigned Reg, + std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, SmallVectorImpl<unsigned> *Ops = nullptr) const; /// Return true if the MachineInstr kills the specified register. /// If TargetRegisterInfo is passed, then it also checks if there is /// a kill of a super-register. - bool killsRegister(unsigned Reg, + bool killsRegister(Register Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterUseOperandIdx(Reg, true, TRI) != -1; } @@ -1170,7 +1189,7 @@ public: /// If TargetRegisterInfo is passed, then it also checks /// if there is a def of a super-register. /// NOTE: It's ignoring subreg indices on virtual registers. - bool definesRegister(unsigned Reg, + bool definesRegister(Register Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; } @@ -1178,38 +1197,38 @@ public: /// Return true if the MachineInstr modifies (fully define or partially /// define) the specified register. /// NOTE: It's ignoring subreg indices on virtual registers. - bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const { + bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const { return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; } /// Returns true if the register is dead in this machine instruction. /// If TargetRegisterInfo is passed, then it also checks /// if there is a dead def of a super-register. - bool registerDefIsDead(unsigned Reg, + bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI = nullptr) const { return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; } /// Returns true if the MachineInstr has an implicit-use operand of exactly /// the given register (not considering sub/super-registers). - bool hasRegisterImplicitUseOperand(unsigned Reg) const; + bool hasRegisterImplicitUseOperand(Register Reg) const; /// Returns the operand index that is a use of the specific register or -1 /// if it is not found. It further tightens the search criteria to a use /// that kills the register if isKill is true. - int findRegisterUseOperandIdx(unsigned Reg, bool isKill = false, + int findRegisterUseOperandIdx(Register Reg, bool isKill = false, const TargetRegisterInfo *TRI = nullptr) const; /// Wrapper for findRegisterUseOperandIdx, it returns /// a pointer to the MachineOperand rather than an index. - MachineOperand *findRegisterUseOperand(unsigned Reg, bool isKill = false, + MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, const TargetRegisterInfo *TRI = nullptr) { int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); return (Idx == -1) ? nullptr : &getOperand(Idx); } const MachineOperand *findRegisterUseOperand( - unsigned Reg, bool isKill = false, + Register Reg, bool isKill = false, const TargetRegisterInfo *TRI = nullptr) const { return const_cast<MachineInstr *>(this)-> findRegisterUseOperand(Reg, isKill, TRI); @@ -1221,14 +1240,14 @@ public: /// overlap the specified register. If TargetRegisterInfo is non-null, /// then it also checks if there is a def of a super-register. /// This may also return a register mask operand when Overlap is true. - int findRegisterDefOperandIdx(unsigned Reg, + int findRegisterDefOperandIdx(Register Reg, bool isDead = false, bool Overlap = false, const TargetRegisterInfo *TRI = nullptr) const; /// Wrapper for findRegisterDefOperandIdx, it returns /// a pointer to the MachineOperand rather than an index. MachineOperand * - findRegisterDefOperand(unsigned Reg, bool isDead = false, + findRegisterDefOperand(Register Reg, bool isDead = false, bool Overlap = false, const TargetRegisterInfo *TRI = nullptr) { int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); @@ -1236,7 +1255,7 @@ public: } const MachineOperand * - findRegisterDefOperand(unsigned Reg, bool isDead = false, + findRegisterDefOperand(Register Reg, bool isDead = false, bool Overlap = false, const TargetRegisterInfo *TRI = nullptr) const { return const_cast<MachineInstr *>(this)->findRegisterDefOperand( @@ -1283,7 +1302,7 @@ public: /// /// \pre CurRC must not be NULL. const TargetRegisterClass *getRegClassConstraintEffectForVReg( - unsigned Reg, const TargetRegisterClass *CurRC, + Register Reg, const TargetRegisterClass *CurRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ExploreBundle = false) const; @@ -1346,39 +1365,39 @@ public: /// Replace all occurrences of FromReg with ToReg:SubIdx, /// properly composing subreg indices where necessary. - void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx, + void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo); /// We have determined MI kills a register. Look for the /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, /// add a implicit operand if it's not found. Returns true if the operand /// exists / is added. - bool addRegisterKilled(unsigned IncomingReg, + bool addRegisterKilled(Register IncomingReg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound = false); /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes /// all aliasing registers. - void clearRegisterKills(unsigned Reg, const TargetRegisterInfo *RegInfo); + void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); /// We have determined MI defined a register without a use. /// Look for the operand that defines it and mark it as IsDead. If /// AddIfNotFound is true, add a implicit operand if it's not found. Returns /// true if the operand exists / is added. - bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo, + bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound = false); /// Clear all dead flags on operands defining register @p Reg. - void clearRegisterDeads(unsigned Reg); + void clearRegisterDeads(Register Reg); /// Mark all subregister defs of register @p Reg with the undef flag. /// This function is used when we determined to have a subregister def in an /// otherwise undefined super register. - void setRegisterDefReadUndef(unsigned Reg, bool IsUndef = true); + void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); /// We have determined MI defines a register. Make sure there is an operand /// defining Reg. - void addRegisterDefined(unsigned Reg, + void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo = nullptr); /// Mark every physreg used by this instruction as @@ -1386,13 +1405,13 @@ public: /// /// On instructions with register mask operands, also add implicit-def /// operands for all registers in UsedRegs. - void setPhysRegsDeadExcept(ArrayRef<unsigned> UsedRegs, + void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, const TargetRegisterInfo &TRI); /// Return true if it is safe to move this instruction. If /// SawStore is set to true, it means that there is a store (or call) between /// the instruction's location and its intended destination. - bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const; + bool isSafeToMove(AAResults *AA, bool &SawStore) const; /// Returns true if this instruction's memory access aliases the memory /// access of Other. @@ -1404,7 +1423,7 @@ public: /// @param AA Optional alias analysis, used to compare memory operands. /// @param Other MachineInstr to check aliasing against. /// @param UseTBAA Whether to pass TBAA information to alias analysis. - bool mayAlias(AliasAnalysis *AA, const MachineInstr &Other, bool UseTBAA) const; + bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; /// Return true if this instruction may have an ordered /// or volatile memory reference, or if the information describing the memory @@ -1419,7 +1438,7 @@ public: /// argument area of a function (if it does not change). If the instruction /// does multiple loads, this returns true only if all of the loads are /// dereferenceable and invariant. - bool isDereferenceableInvariantLoad(AliasAnalysis *AA) const; + bool isDereferenceableInvariantLoad(AAResults *AA) const; /// If the specified instruction is a PHI that always merges together the /// same virtual register, return the register, otherwise return 0. @@ -1603,9 +1622,15 @@ public: /// Scan instructions following MI and collect any matching DBG_VALUEs. void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); - /// Find all DBG_VALUEs immediately following this instruction that point - /// to a register def in this instruction and point them to \p Reg instead. - void changeDebugValuesDefReg(unsigned Reg); + /// Find all DBG_VALUEs that point to the register def in this instruction + /// and point them to \p Reg instead. + void changeDebugValuesDefReg(Register Reg); + + /// Returns the Intrinsic::ID for this instruction. + /// \pre Must have an intrinsic ID operand. + unsigned getIntrinsicID() const { + return getOperand(getNumExplicitDefs()).getIntrinsicID(); + } private: /// If this instruction is embedded into a MachineFunction, return the @@ -1630,7 +1655,7 @@ private: /// this MI and the given operand index \p OpIdx. /// If the related operand does not constrained Reg, this returns CurRC. const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( - unsigned OpIdx, unsigned Reg, const TargetRegisterClass *CurRC, + unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; }; diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index 6d7fb72b6bd1..880d4829ac7e 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -85,7 +85,7 @@ public: Register getReg(unsigned Idx) const { return MI->getOperand(Idx).getReg(); } /// Add a new virtual register operand. - const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0, + const MachineInstrBuilder &addReg(Register RegNo, unsigned flags = 0, unsigned SubReg = 0) const { assert((flags & 0x1) == 0 && "Passing in 'true' to addReg is forbidden! Use enums instead."); @@ -104,14 +104,14 @@ public: } /// Add a virtual register definition operand. - const MachineInstrBuilder &addDef(unsigned RegNo, unsigned Flags = 0, + const MachineInstrBuilder &addDef(Register RegNo, unsigned Flags = 0, unsigned SubReg = 0) const { return addReg(RegNo, Flags | RegState::Define, SubReg); } /// Add a virtual register use operand. It is an error for Flags to contain /// `RegState::Define` when calling this function. - const MachineInstrBuilder &addUse(unsigned RegNo, unsigned Flags = 0, + const MachineInstrBuilder &addUse(Register RegNo, unsigned Flags = 0, unsigned SubReg = 0) const { assert(!(Flags & RegState::Define) && "Misleading addUse defines register, use addReg instead."); @@ -135,7 +135,7 @@ public: } const MachineInstrBuilder &addMBB(MachineBasicBlock *MBB, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateMBB(MBB, TargetFlags)); return *this; } @@ -145,42 +145,42 @@ public: return *this; } - const MachineInstrBuilder &addConstantPoolIndex(unsigned Idx, - int Offset = 0, - unsigned char TargetFlags = 0) const { + const MachineInstrBuilder & + addConstantPoolIndex(unsigned Idx, int Offset = 0, + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateCPI(Idx, Offset, TargetFlags)); return *this; } const MachineInstrBuilder &addTargetIndex(unsigned Idx, int64_t Offset = 0, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateTargetIndex(Idx, Offset, TargetFlags)); return *this; } const MachineInstrBuilder &addJumpTableIndex(unsigned Idx, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateJTI(Idx, TargetFlags)); return *this; } const MachineInstrBuilder &addGlobalAddress(const GlobalValue *GV, int64_t Offset = 0, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateGA(GV, Offset, TargetFlags)); return *this; } const MachineInstrBuilder &addExternalSymbol(const char *FnName, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateES(FnName, TargetFlags)); return *this; } const MachineInstrBuilder &addBlockAddress(const BlockAddress *BA, int64_t Offset = 0, - unsigned char TargetFlags = 0) const { + unsigned TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateBA(BA, Offset, TargetFlags)); return *this; } @@ -250,6 +250,11 @@ public: return *this; } + const MachineInstrBuilder &addShuffleMask(const Constant *Val) const { + MI->addOperand(*MF, MachineOperand::CreateShuffleMask(Val)); + return *this; + } + const MachineInstrBuilder &addSym(MCSymbol *Sym, unsigned char TargetFlags = 0) const { MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym, TargetFlags)); @@ -316,7 +321,7 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, /// This version of the builder sets up the first operand as a /// destination virtual register. inline MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, - const MCInstrDesc &MCID, unsigned DestReg) { + const MCInstrDesc &MCID, Register DestReg) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, DL)) .addReg(DestReg, RegState::Define); } @@ -327,7 +332,7 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, const DebugLoc &DL, const MCInstrDesc &MCID, - unsigned DestReg) { + Register DestReg) { MachineFunction &MF = *BB.getParent(); MachineInstr *MI = MF.CreateMachineInstr(MCID, DL); BB.insert(I, MI); @@ -343,7 +348,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::instr_iterator I, const DebugLoc &DL, const MCInstrDesc &MCID, - unsigned DestReg) { + Register DestReg) { MachineFunction &MF = *BB.getParent(); MachineInstr *MI = MF.CreateMachineInstr(MCID, DL); BB.insert(I, MI); @@ -352,7 +357,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr &I, const DebugLoc &DL, const MCInstrDesc &MCID, - unsigned DestReg) { + Register DestReg) { // Calling the overload for instr_iterator is always correct. However, the // definition is not available in headers, so inline the check. if (I.isInsideBundle()) @@ -362,7 +367,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr &I, inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *I, const DebugLoc &DL, const MCInstrDesc &MCID, - unsigned DestReg) { + Register DestReg) { return BuildMI(BB, *I, DL, MCID, DestReg); } @@ -416,7 +421,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, const DebugLoc &DL, /// end of the given MachineBasicBlock, and sets up the first operand as a /// destination virtual register. inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, const DebugLoc &DL, - const MCInstrDesc &MCID, unsigned DestReg) { + const MCInstrDesc &MCID, Register DestReg) { return BuildMI(*BB, BB->end(), DL, MCID, DestReg); } @@ -426,7 +431,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, const DebugLoc &DL, /// second operand is an immediate. MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID, bool IsIndirect, - unsigned Reg, const MDNode *Variable, + Register Reg, const MDNode *Variable, const MDNode *Expr); /// This version of the builder builds a DBG_VALUE intrinsic @@ -442,7 +447,7 @@ MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, const DebugLoc &DL, const MCInstrDesc &MCID, bool IsIndirect, - unsigned Reg, const MDNode *Variable, + Register Reg, const MDNode *Variable, const MDNode *Expr); /// This version of the builder builds a DBG_VALUE intrinsic @@ -490,16 +495,13 @@ inline unsigned getRenamableRegState(bool B) { /// Get all register state flags from machine operand \p RegOp. inline unsigned getRegState(const MachineOperand &RegOp) { assert(RegOp.isReg() && "Not a register operand"); - return getDefRegState(RegOp.isDef()) | - getImplRegState(RegOp.isImplicit()) | - getKillRegState(RegOp.isKill()) | - getDeadRegState(RegOp.isDead()) | - getUndefRegState(RegOp.isUndef()) | - getInternalReadRegState(RegOp.isInternalRead()) | - getDebugRegState(RegOp.isDebug()) | - getRenamableRegState( - TargetRegisterInfo::isPhysicalRegister(RegOp.getReg()) && - RegOp.isRenamable()); + return getDefRegState(RegOp.isDef()) | getImplRegState(RegOp.isImplicit()) | + getKillRegState(RegOp.isKill()) | getDeadRegState(RegOp.isDead()) | + getUndefRegState(RegOp.isUndef()) | + getInternalReadRegState(RegOp.isInternalRead()) | + getDebugRegState(RegOp.isDebug()) | + getRenamableRegState(Register::isPhysicalRegister(RegOp.getReg()) && + RegOp.isRenamable()); } /// Helper class for constructing bundles of MachineInstrs. diff --git a/include/llvm/CodeGen/MachineLoopUtils.h b/include/llvm/CodeGen/MachineLoopUtils.h new file mode 100644 index 000000000000..41379b75d00a --- /dev/null +++ b/include/llvm/CodeGen/MachineLoopUtils.h @@ -0,0 +1,41 @@ +//=- MachineLoopUtils.h - Helper functions for manipulating loops -*- C++ -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H +#define LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H + +namespace llvm { +class MachineBasicBlock; +class MachineRegisterInfo; +class TargetInstrInfo; + +enum LoopPeelDirection { + LPD_Front, ///< Peel the first iteration of the loop. + LPD_Back ///< Peel the last iteration of the loop. +}; + +/// Peels a single block loop. Loop must have two successors, one of which +/// must be itself. Similarly it must have two predecessors, one of which must +/// be itself. +/// +/// The loop block is copied and inserted into the CFG such that two copies of +/// the loop follow on from each other. The copy is inserted either before or +/// after the loop based on Direction. +/// +/// Phis are updated and an unconditional branch inserted at the end of the +/// clone so as to execute a single iteration. +/// +/// The trip count of Loop is not updated. +MachineBasicBlock *PeelSingleBlockLoop(LoopPeelDirection Direction, + MachineBasicBlock *Loop, + MachineRegisterInfo &MRI, + const TargetInstrInfo *TII); + +} // namespace llvm + +#endif // LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index 65f706302bc2..33a48a235e18 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -293,8 +293,6 @@ public: /// Support for operator<<. /// @{ - void print(raw_ostream &OS) const; - void print(raw_ostream &OS, ModuleSlotTracker &MST) const; void print(raw_ostream &OS, ModuleSlotTracker &MST, SmallVectorImpl<StringRef> &SSNs, const LLVMContext &Context, const MachineFrameInfo *MFI, const TargetInstrInfo *TII) const; @@ -319,11 +317,6 @@ public: } }; -inline raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO) { - MRO.print(OS); - return OS; -} - } // End llvm namespace #endif diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 4ff5c7fd013a..6902dada2423 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -33,6 +33,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerIntPair.h" +#include "llvm/IR/PassManager.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Pass.h" @@ -74,7 +75,10 @@ protected: /// made by different debugging and exception handling schemes and reformated /// for specific use. /// -class MachineModuleInfo : public ImmutablePass { +class MachineModuleInfo { + friend class MachineModuleInfoWrapperPass; + friend class MachineModuleAnalysis; + const LLVMTargetMachine &TM; /// This is the MCContext used for the entire code generator. @@ -140,15 +144,17 @@ class MachineModuleInfo : public ImmutablePass { const Function *LastRequest = nullptr; ///< Used for shortcut/cache. MachineFunction *LastResult = nullptr; ///< Used for shortcut/cache. -public: - static char ID; // Pass identification, replacement for typeid + MachineModuleInfo &operator=(MachineModuleInfo &&MMII) = delete; +public: explicit MachineModuleInfo(const LLVMTargetMachine *TM = nullptr); - ~MachineModuleInfo() override; - // Initialization and Finalization - bool doInitialization(Module &) override; - bool doFinalization(Module &) override; + MachineModuleInfo(MachineModuleInfo &&MMII); + + ~MachineModuleInfo(); + + void initialize(); + void finalize(); const LLVMTargetMachine &getTarget() const { return TM; } @@ -254,6 +260,38 @@ public: /// \} }; // End class MachineModuleInfo +class MachineModuleInfoWrapperPass : public ImmutablePass { + MachineModuleInfo MMI; + +public: + static char ID; // Pass identification, replacement for typeid + explicit MachineModuleInfoWrapperPass(const LLVMTargetMachine *TM = nullptr); + + // Initialization and Finalization + bool doInitialization(Module &) override; + bool doFinalization(Module &) override; + + MachineModuleInfo &getMMI() { return MMI; } + const MachineModuleInfo &getMMI() const { return MMI; } +}; + +/// An analysis that produces \c MachineInfo for a module. +class MachineModuleAnalysis : public AnalysisInfoMixin<MachineModuleAnalysis> { + friend AnalysisInfoMixin<MachineModuleAnalysis>; + static AnalysisKey Key; + + const LLVMTargetMachine *TM; + +public: + /// Provide the result type for this analysis pass. + using Result = MachineModuleInfo; + + MachineModuleAnalysis(const LLVMTargetMachine *TM) : TM(TM) {} + + /// Run the analysis pass and produce machine module information. + MachineModuleInfo run(Module &M, ModuleAnalysisManager &); +}; + } // end namespace llvm #endif // LLVM_CODEGEN_MACHINEMODULEINFO_H diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h index 2152c7582e5a..df914dc2d85e 100644 --- a/include/llvm/CodeGen/MachineOperand.h +++ b/include/llvm/CodeGen/MachineOperand.h @@ -23,6 +23,7 @@ namespace llvm { class BlockAddress; +class Constant; class ConstantFP; class ConstantInt; class GlobalValue; @@ -68,7 +69,8 @@ public: MO_CFIIndex, ///< MCCFIInstruction index. MO_IntrinsicID, ///< Intrinsic ID for ISel MO_Predicate, ///< Generic predicate for ISel - MO_Last = MO_Predicate, + MO_ShuffleMask, ///< Other IR Constant for ISel (shuffle masks) + MO_Last = MO_ShuffleMask }; private: @@ -172,6 +174,7 @@ private: unsigned CFIIndex; // For MO_CFI. Intrinsic::ID IntrinsicID; // For MO_IntrinsicID. unsigned Pred; // For MO_Predicate + const Constant *ShuffleMask; // For MO_ShuffleMask struct { // For MO_Register. // Register number is in SmallContents.RegNo. @@ -341,6 +344,7 @@ public: bool isCFIIndex() const { return OpKind == MO_CFIIndex; } bool isIntrinsicID() const { return OpKind == MO_IntrinsicID; } bool isPredicate() const { return OpKind == MO_Predicate; } + bool isShuffleMask() const { return OpKind == MO_ShuffleMask; } //===--------------------------------------------------------------------===// // Accessors for Register Operands //===--------------------------------------------------------------------===// @@ -455,7 +459,7 @@ public: /// Change the register this operand corresponds to. /// - void setReg(unsigned Reg); + void setReg(Register Reg); void setSubReg(unsigned subReg) { assert(isReg() && "Wrong MachineOperand mutator"); @@ -468,13 +472,13 @@ public: /// using TargetRegisterInfo to compose the subreg indices if necessary. /// Reg must be a virtual register, SubIdx can be 0. /// - void substVirtReg(unsigned Reg, unsigned SubIdx, const TargetRegisterInfo&); + void substVirtReg(Register Reg, unsigned SubIdx, const TargetRegisterInfo&); /// substPhysReg - Substitute the current register with the physical register /// Reg, taking any existing SubReg into account. For instance, /// substPhysReg(%eax) will change %reg1024:sub_8bit to %al. /// - void substPhysReg(unsigned Reg, const TargetRegisterInfo&); + void substPhysReg(MCRegister Reg, const TargetRegisterInfo&); void setIsUse(bool Val = true) { setIsDef(!Val); } @@ -579,6 +583,11 @@ public: return Contents.Pred; } + const Constant *getShuffleMask() const { + assert(isShuffleMask() && "Wrong MachineOperand accessor"); + return Contents.ShuffleMask; + } + /// Return the offset from the symbol in this operand. This always returns 0 /// for ExternalSymbol operands. int64_t getOffset() const { @@ -717,11 +726,11 @@ public: void ChangeToFPImmediate(const ConstantFP *FPImm); /// ChangeToES - Replace this operand with a new external symbol operand. - void ChangeToES(const char *SymName, unsigned char TargetFlags = 0); + void ChangeToES(const char *SymName, unsigned TargetFlags = 0); /// ChangeToGA - Replace this operand with a new global address operand. void ChangeToGA(const GlobalValue *GV, int64_t Offset, - unsigned char TargetFlags = 0); + unsigned TargetFlags = 0); /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand. void ChangeToMCSymbol(MCSymbol *Sym); @@ -731,12 +740,12 @@ public: /// Replace this operand with a target index. void ChangeToTargetIndex(unsigned Idx, int64_t Offset, - unsigned char TargetFlags = 0); + unsigned TargetFlags = 0); /// ChangeToRegister - Replace this operand with a new register operand of /// the specified value. If an operand is known to be an register already, /// the setReg method should be used. - void ChangeToRegister(unsigned Reg, bool isDef, bool isImp = false, + void ChangeToRegister(Register Reg, bool isDef, bool isImp = false, bool isKill = false, bool isDead = false, bool isUndef = false, bool isDebug = false); @@ -762,7 +771,7 @@ public: return Op; } - static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp = false, + static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp = false, bool isKill = false, bool isDead = false, bool isUndef = false, bool isEarlyClobber = false, @@ -788,7 +797,7 @@ public: return Op; } static MachineOperand CreateMBB(MachineBasicBlock *MBB, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_MachineBasicBlock); Op.setMBB(MBB); Op.setTargetFlags(TargetFlags); @@ -800,7 +809,7 @@ public: return Op; } static MachineOperand CreateCPI(unsigned Idx, int Offset, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_ConstantPoolIndex); Op.setIndex(Idx); Op.setOffset(Offset); @@ -808,21 +817,21 @@ public: return Op; } static MachineOperand CreateTargetIndex(unsigned Idx, int64_t Offset, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_TargetIndex); Op.setIndex(Idx); Op.setOffset(Offset); Op.setTargetFlags(TargetFlags); return Op; } - static MachineOperand CreateJTI(unsigned Idx, unsigned char TargetFlags = 0) { + static MachineOperand CreateJTI(unsigned Idx, unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_JumpTableIndex); Op.setIndex(Idx); Op.setTargetFlags(TargetFlags); return Op; } static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_GlobalAddress); Op.Contents.OffsetedInfo.Val.GV = GV; Op.setOffset(Offset); @@ -830,7 +839,7 @@ public: return Op; } static MachineOperand CreateES(const char *SymName, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_ExternalSymbol); Op.Contents.OffsetedInfo.Val.SymbolName = SymName; Op.setOffset(0); // Offset is always 0. @@ -838,7 +847,7 @@ public: return Op; } static MachineOperand CreateBA(const BlockAddress *BA, int64_t Offset, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_BlockAddress); Op.Contents.OffsetedInfo.Val.BA = BA; Op.setOffset(Offset); @@ -876,7 +885,7 @@ public: } static MachineOperand CreateMCSymbol(MCSymbol *Sym, - unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_MCSymbol); Op.Contents.Sym = Sym; Op.setOffset(0); @@ -902,6 +911,12 @@ public: return Op; } + static MachineOperand CreateShuffleMask(const Constant *C) { + MachineOperand Op(MachineOperand::MO_ShuffleMask); + Op.Contents.ShuffleMask = C; + return Op; + } + friend class MachineInstr; friend class MachineRegisterInfo; diff --git a/include/llvm/CodeGen/MachinePipeliner.h b/include/llvm/CodeGen/MachinePipeliner.h index 03ca53072685..e9cf7e115bff 100644 --- a/include/llvm/CodeGen/MachinePipeliner.h +++ b/include/llvm/CodeGen/MachinePipeliner.h @@ -40,6 +40,8 @@ #ifndef LLVM_LIB_CODEGEN_MACHINEPIPELINER_H #define LLVM_LIB_CODEGEN_MACHINEPIPELINER_H +#include "llvm/Analysis/AliasAnalysis.h" + #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" @@ -148,7 +150,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs { /// We may create a new instruction, so remember it because it /// must be deleted when the pass is finished. - SmallPtrSet<MachineInstr *, 4> NewMIs; + DenseMap<MachineInstr*, MachineInstr *> NewMIs; /// Ordered list of DAG postprocessing steps. std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations; @@ -200,7 +202,7 @@ public: RegClassInfo(rci), II_setByPragma(II), Topo(SUnits, &ExitSU) { P.MF->getSubtarget().getSMSMutations(Mutations); if (SwpEnableCopyToPhi) - Mutations.push_back(llvm::make_unique<CopyToPhiMutation>()); + Mutations.push_back(std::make_unique<CopyToPhiMutation>()); } void schedule() override; @@ -297,53 +299,8 @@ private: void computeNodeOrder(NodeSetType &NodeSets); void checkValidNodeOrder(const NodeSetType &Circuits) const; bool schedulePipeline(SMSchedule &Schedule); - void generatePipelinedLoop(SMSchedule &Schedule); - void generateProlog(SMSchedule &Schedule, unsigned LastStage, - MachineBasicBlock *KernelBB, ValueMapTy *VRMap, - MBBVectorTy &PrologBBs); - void generateEpilog(SMSchedule &Schedule, unsigned LastStage, - MachineBasicBlock *KernelBB, ValueMapTy *VRMap, - MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs); - void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, - MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, - SMSchedule &Schedule, ValueMapTy *VRMap, - InstrMapTy &InstrMap, unsigned LastStageNum, - unsigned CurStageNum, bool IsLast); - void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, - MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, - SMSchedule &Schedule, ValueMapTy *VRMap, - InstrMapTy &InstrMap, unsigned LastStageNum, - unsigned CurStageNum, bool IsLast); - void removeDeadInstructions(MachineBasicBlock *KernelBB, - MBBVectorTy &EpilogBBs); - void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, - SMSchedule &Schedule); - void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs, - MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, - SMSchedule &Schedule, ValueMapTy *VRMap); bool computeDelta(MachineInstr &MI, unsigned &Delta); - void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI, - unsigned Num); - MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum, - unsigned InstStageNum); - MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum, - unsigned InstStageNum, - SMSchedule &Schedule); - void updateInstruction(MachineInstr *NewMI, bool LastDef, - unsigned CurStageNum, unsigned InstrStageNum, - SMSchedule &Schedule, ValueMapTy *VRMap); MachineInstr *findDefInLoop(unsigned Reg); - unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal, - unsigned LoopStage, ValueMapTy *VRMap, - MachineBasicBlock *BB); - void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum, - SMSchedule &Schedule, ValueMapTy *VRMap, - InstrMapTy &InstrMap); - void rewriteScheduledInstr(MachineBasicBlock *BB, SMSchedule &Schedule, - InstrMapTy &InstrMap, unsigned CurStageNum, - unsigned PhiNum, MachineInstr *Phi, - unsigned OldReg, unsigned NewReg, - unsigned PrevReg = 0); bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos, unsigned &OffsetPos, unsigned &NewBase, int64_t &NewOffset); @@ -529,12 +486,6 @@ private: /// Map from instruction to execution cycle. std::map<SUnit *, int> InstrToCycle; - /// Map for each register and the max difference between its uses and def. - /// The first element in the pair is the max difference in stages. The - /// second is true if the register defines a Phi value and loop value is - /// scheduled before the Phi. - std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff; - /// Keep track of the first cycle value in the schedule. It starts /// as zero, but the algorithm allows negative values. int FirstCycle = 0; @@ -560,7 +511,6 @@ public: void reset() { ScheduledInstrs.clear(); InstrToCycle.clear(); - RegToStageDiff.clear(); FirstCycle = 0; LastCycle = 0; InitiationInterval = 0; @@ -620,28 +570,6 @@ public: return (LastCycle - FirstCycle) / InitiationInterval; } - /// Return the max. number of stages/iterations that can occur between a - /// register definition and its uses. - unsigned getStagesForReg(int Reg, unsigned CurStage) { - std::pair<unsigned, bool> Stages = RegToStageDiff[Reg]; - if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second) - return 1; - return Stages.first; - } - - /// The number of stages for a Phi is a little different than other - /// instructions. The minimum value computed in RegToStageDiff is 1 - /// because we assume the Phi is needed for at least 1 iteration. - /// This is not the case if the loop value is scheduled prior to the - /// Phi in the same stage. This function returns the number of stages - /// or iterations needed between the Phi definition and any uses. - unsigned getStagesForPhi(int Reg) { - std::pair<unsigned, bool> Stages = RegToStageDiff[Reg]; - if (Stages.second) - return Stages.first; - return Stages.first - 1; - } - /// Return the instructions that are scheduled at the specified cycle. std::deque<SUnit *> &getInstructions(int cycle) { return ScheduledInstrs[cycle]; diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h index b67e6b52ac8f..cb258b5e7b21 100644 --- a/include/llvm/CodeGen/MachinePostDominators.h +++ b/include/llvm/CodeGen/MachinePostDominators.h @@ -16,68 +16,76 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include <memory> namespace llvm { /// -/// PostDominatorTree Class - Concrete subclass of DominatorTree that is used -/// to compute the post-dominator tree. +/// MachinePostDominatorTree - an analysis pass wrapper for DominatorTree +/// used to compute the post-dominator tree for MachineFunctions. /// -struct MachinePostDominatorTree : public MachineFunctionPass { -private: - PostDomTreeBase<MachineBasicBlock> *DT; +class MachinePostDominatorTree : public MachineFunctionPass { + using PostDomTreeT = PostDomTreeBase<MachineBasicBlock>; + std::unique_ptr<PostDomTreeT> PDT; public: static char ID; MachinePostDominatorTree(); - ~MachinePostDominatorTree() override; - FunctionPass *createMachinePostDominatorTreePass(); const SmallVectorImpl<MachineBasicBlock *> &getRoots() const { - return DT->getRoots(); + return PDT->getRoots(); } - MachineDomTreeNode *getRootNode() const { - return DT->getRootNode(); - } + MachineDomTreeNode *getRootNode() const { return PDT->getRootNode(); } MachineDomTreeNode *operator[](MachineBasicBlock *BB) const { - return DT->getNode(BB); + return PDT->getNode(BB); } MachineDomTreeNode *getNode(MachineBasicBlock *BB) const { - return DT->getNode(BB); + return PDT->getNode(BB); } bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const { - return DT->dominates(A, B); + return PDT->dominates(A, B); } bool dominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const { - return DT->dominates(A, B); + return PDT->dominates(A, B); } bool properlyDominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const { - return DT->properlyDominates(A, B); + return PDT->properlyDominates(A, B); } bool properlyDominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const { - return DT->properlyDominates(A, B); + return PDT->properlyDominates(A, B); + } + + bool isVirtualRoot(const MachineDomTreeNode *Node) const { + return PDT->isVirtualRoot(Node); } MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A, - MachineBasicBlock *B) { - return DT->findNearestCommonDominator(A, B); + MachineBasicBlock *B) const { + return PDT->findNearestCommonDominator(A, B); } + /// Returns the nearest common dominator of the given blocks. + /// If that tree node is a virtual root, a nullptr will be returned. + MachineBasicBlock * + findNearestCommonDominator(ArrayRef<MachineBasicBlock *> Blocks) const; + bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override { PDT.reset(nullptr); } + void verifyAnalysis() const override; void print(llvm::raw_ostream &OS, const Module *M = nullptr) const override; }; } //end of namespace llvm diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h index 6d9fb9b9100a..eeb69fef2c6b 100644 --- a/include/llvm/CodeGen/MachineRegionInfo.h +++ b/include/llvm/CodeGen/MachineRegionInfo.h @@ -22,7 +22,7 @@ namespace llvm { -struct MachinePostDominatorTree; +class MachinePostDominatorTree; class MachineRegion; class MachineRegionNode; class MachineRegionInfo; diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index b5deed1f5010..488a5a55a169 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -107,16 +107,16 @@ private: /// getRegUseDefListHead - Return the head pointer for the register use/def /// list for the specified virtual or physical register. - MachineOperand *&getRegUseDefListHead(unsigned RegNo) { - if (TargetRegisterInfo::isVirtualRegister(RegNo)) - return VRegInfo[RegNo].second; - return PhysRegUseDefLists[RegNo]; + MachineOperand *&getRegUseDefListHead(Register RegNo) { + if (RegNo.isVirtual()) + return VRegInfo[RegNo.id()].second; + return PhysRegUseDefLists[RegNo.id()]; } - MachineOperand *getRegUseDefListHead(unsigned RegNo) const { - if (TargetRegisterInfo::isVirtualRegister(RegNo)) - return VRegInfo[RegNo].second; - return PhysRegUseDefLists[RegNo]; + MachineOperand *getRegUseDefListHead(Register RegNo) const { + if (RegNo.isVirtual()) + return VRegInfo[RegNo.id()].second; + return PhysRegUseDefLists[RegNo.id()]; } /// Get the next element in the use-def chain. @@ -214,8 +214,8 @@ public: bool shouldTrackSubRegLiveness(const TargetRegisterClass &RC) const { return subRegLivenessEnabled() && RC.HasDisjunctSubRegs; } - bool shouldTrackSubRegLiveness(unsigned VReg) const { - assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Must pass a VReg"); + bool shouldTrackSubRegLiveness(Register VReg) const { + assert(VReg.isVirtual() && "Must pass a VReg"); return shouldTrackSubRegLiveness(*getRegClass(VReg)); } bool subRegLivenessEnabled() const { @@ -326,7 +326,7 @@ public: /// of the specified register, skipping those marked as Debug. using reg_nodbg_iterator = defusechain_iterator<true, true, true, true, false, false>; - reg_nodbg_iterator reg_nodbg_begin(unsigned RegNo) const { + reg_nodbg_iterator reg_nodbg_begin(Register RegNo) const { return reg_nodbg_iterator(getRegUseDefListHead(RegNo)); } static reg_nodbg_iterator reg_nodbg_end() { @@ -374,7 +374,7 @@ public: /// reg_nodbg_empty - Return true if the only instructions using or defining /// Reg are Debug instructions. - bool reg_nodbg_empty(unsigned RegNo) const { + bool reg_nodbg_empty(Register RegNo) const { return reg_nodbg_begin(RegNo) == reg_nodbg_end(); } @@ -628,10 +628,10 @@ public: /// Return the register class of the specified virtual register. /// This shouldn't be used directly unless \p Reg has a register class. /// \see getRegClassOrNull when this might happen. - const TargetRegisterClass *getRegClass(unsigned Reg) const { - assert(VRegInfo[Reg].first.is<const TargetRegisterClass *>() && + const TargetRegisterClass *getRegClass(Register Reg) const { + assert(VRegInfo[Reg.id()].first.is<const TargetRegisterClass *>() && "Register class not set, wrong accessor"); - return VRegInfo[Reg].first.get<const TargetRegisterClass *>(); + return VRegInfo[Reg.id()].first.get<const TargetRegisterClass *>(); } /// Return the register class of \p Reg, or null if Reg has not been assigned @@ -727,7 +727,7 @@ public: /// Get the low-level type of \p Reg or LLT{} if Reg is not a generic /// (target independent) virtual register. LLT getType(unsigned Reg) const { - if (TargetRegisterInfo::isVirtualRegister(Reg) && VRegToType.inBounds(Reg)) + if (Register::isVirtualRegister(Reg) && VRegToType.inBounds(Reg)) return VRegToType[Reg]; return LLT{}; } @@ -760,7 +760,7 @@ public: /// specified virtual register. This is typically used by target, and in case /// of an earlier hint it will be overwritten. void setRegAllocationHint(unsigned VReg, unsigned Type, unsigned PrefReg) { - assert(TargetRegisterInfo::isVirtualRegister(VReg)); + assert(Register::isVirtualRegister(VReg)); RegAllocHints[VReg].first = Type; RegAllocHints[VReg].second.clear(); RegAllocHints[VReg].second.push_back(PrefReg); @@ -769,7 +769,7 @@ public: /// addRegAllocationHint - Add a register allocation hint to the hints /// vector for VReg. void addRegAllocationHint(unsigned VReg, unsigned PrefReg) { - assert(TargetRegisterInfo::isVirtualRegister(VReg)); + assert(Register::isVirtualRegister(VReg)); RegAllocHints[VReg].second.push_back(PrefReg); } @@ -789,17 +789,18 @@ public: /// specified virtual register. If there are many hints, this returns the /// one with the greatest weight. std::pair<unsigned, unsigned> - getRegAllocationHint(unsigned VReg) const { - assert(TargetRegisterInfo::isVirtualRegister(VReg)); - unsigned BestHint = (RegAllocHints[VReg].second.size() ? - RegAllocHints[VReg].second[0] : 0); - return std::pair<unsigned, unsigned>(RegAllocHints[VReg].first, BestHint); + getRegAllocationHint(Register VReg) const { + assert(VReg.isVirtual()); + unsigned BestHint = (RegAllocHints[VReg.id()].second.size() ? + RegAllocHints[VReg.id()].second[0] : 0); + return std::pair<unsigned, unsigned>(RegAllocHints[VReg.id()].first, + BestHint); } /// getSimpleHint - same as getRegAllocationHint except it will only return /// a target independent hint. - unsigned getSimpleHint(unsigned VReg) const { - assert(TargetRegisterInfo::isVirtualRegister(VReg)); + Register getSimpleHint(Register VReg) const { + assert(VReg.isVirtual()); std::pair<unsigned, unsigned> Hint = getRegAllocationHint(VReg); return Hint.first ? 0 : Hint.second; } @@ -808,7 +809,7 @@ public: /// register allocation hints for VReg. const std::pair<unsigned, SmallVector<unsigned, 4>> &getRegAllocationHints(unsigned VReg) const { - assert(TargetRegisterInfo::isVirtualRegister(VReg)); + assert(Register::isVirtualRegister(VReg)); return RegAllocHints[VReg]; } @@ -817,6 +818,17 @@ public: /// deleted during LiveDebugVariables analysis. void markUsesInDebugValueAsUndef(unsigned Reg) const; + /// updateDbgUsersToReg - Update a collection of DBG_VALUE instructions + /// to refer to the designated register. + void updateDbgUsersToReg(unsigned Reg, + ArrayRef<MachineInstr*> Users) const { + for (MachineInstr *MI : Users) { + assert(MI->isDebugInstr()); + assert(MI->getOperand(0).isReg()); + MI->getOperand(0).setReg(Reg); + } + } + /// Return true if the specified register is modified in this function. /// This checks that no defining machine operands exist for the register or /// any of its aliases. Definitions found on functions marked noreturn are @@ -882,8 +894,8 @@ public: /// /// Reserved registers may belong to an allocatable register class, but the /// target has explicitly requested that they are not used. - bool isReserved(unsigned PhysReg) const { - return getReservedRegs().test(PhysReg); + bool isReserved(Register PhysReg) const { + return getReservedRegs().test(PhysReg.id()); } /// Returns true when the given register unit is considered reserved. @@ -1164,7 +1176,7 @@ public: PSetIterator(unsigned RegUnit, const MachineRegisterInfo *MRI) { const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo(); - if (TargetRegisterInfo::isVirtualRegister(RegUnit)) { + if (Register::isVirtualRegister(RegUnit)) { const TargetRegisterClass *RC = MRI->getRegClass(RegUnit); PSet = TRI->getRegClassPressureSets(RC); Weight = TRI->getRegClassWeight(RC).RegWeight; diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 75a334f61ad0..333367943ac0 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -100,6 +100,7 @@ namespace llvm { extern cl::opt<bool> ForceTopDown; extern cl::opt<bool> ForceBottomUp; +extern cl::opt<bool> VerifyScheduling; class LiveIntervals; class MachineDominatorTree; diff --git a/include/llvm/CodeGen/ModuloSchedule.h b/include/llvm/CodeGen/ModuloSchedule.h new file mode 100644 index 000000000000..81a9b63b64ca --- /dev/null +++ b/include/llvm/CodeGen/ModuloSchedule.h @@ -0,0 +1,367 @@ +//===- ModuloSchedule.h - Software pipeline schedule expansion ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Software pipelining (SWP) is an instruction scheduling technique for loops +// that overlaps loop iterations and exploits ILP via compiler transformations. +// +// There are multiple methods for analyzing a loop and creating a schedule. +// An example algorithm is Swing Modulo Scheduling (implemented by the +// MachinePipeliner). The details of how a schedule is arrived at are irrelevant +// for the task of actually rewriting a loop to adhere to the schedule, which +// is what this file does. +// +// A schedule is, for every instruction in a block, a Cycle and a Stage. Note +// that we only support single-block loops, so "block" and "loop" can be used +// interchangably. +// +// The Cycle of an instruction defines a partial order of the instructions in +// the remapped loop. Instructions within a cycle must not consume the output +// of any instruction in the same cycle. Cycle information is assumed to have +// been calculated such that the processor will execute instructions in +// lock-step (for example in a VLIW ISA). +// +// The Stage of an instruction defines the mapping between logical loop +// iterations and pipelined loop iterations. An example (unrolled) pipeline +// may look something like: +// +// I0[0] Execute instruction I0 of iteration 0 +// I1[0], I0[1] Execute I0 of iteration 1 and I1 of iteration 1 +// I1[1], I0[2] +// I1[2], I0[3] +// +// In the schedule for this unrolled sequence we would say that I0 was scheduled +// in stage 0 and I1 in stage 1: +// +// loop: +// [stage 0] x = I0 +// [stage 1] I1 x (from stage 0) +// +// And to actually generate valid code we must insert a phi: +// +// loop: +// x' = phi(x) +// x = I0 +// I1 x' +// +// This is a simple example; the rules for how to generate correct code given +// an arbitrary schedule containing loop-carried values are complex. +// +// Note that these examples only mention the steady-state kernel of the +// generated loop; prologs and epilogs must be generated also that prime and +// flush the pipeline. Doing so is nontrivial. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_MODULOSCHEDULE_H +#define LLVM_LIB_CODEGEN_MODULOSCHEDULE_H + +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineLoopUtils.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include <deque> +#include <vector> + +namespace llvm { +class MachineBasicBlock; +class MachineInstr; +class LiveIntervals; + +/// Represents a schedule for a single-block loop. For every instruction we +/// maintain a Cycle and Stage. +class ModuloSchedule { +private: + /// The block containing the loop instructions. + MachineLoop *Loop; + + /// The instructions to be generated, in total order. Cycle provides a partial + /// order; the total order within cycles has been decided by the schedule + /// producer. + std::vector<MachineInstr *> ScheduledInstrs; + + /// The cycle for each instruction. + DenseMap<MachineInstr *, int> Cycle; + + /// The stage for each instruction. + DenseMap<MachineInstr *, int> Stage; + + /// The number of stages in this schedule (Max(Stage) + 1). + int NumStages; + +public: + /// Create a new ModuloSchedule. + /// \arg ScheduledInstrs The new loop instructions, in total resequenced + /// order. + /// \arg Cycle Cycle index for all instructions in ScheduledInstrs. Cycle does + /// not need to start at zero. ScheduledInstrs must be partially ordered by + /// Cycle. + /// \arg Stage Stage index for all instructions in ScheduleInstrs. + ModuloSchedule(MachineFunction &MF, MachineLoop *Loop, + std::vector<MachineInstr *> ScheduledInstrs, + DenseMap<MachineInstr *, int> Cycle, + DenseMap<MachineInstr *, int> Stage) + : Loop(Loop), ScheduledInstrs(ScheduledInstrs), Cycle(std::move(Cycle)), + Stage(std::move(Stage)) { + NumStages = 0; + for (auto &KV : this->Stage) + NumStages = std::max(NumStages, KV.second); + ++NumStages; + } + + /// Return the single-block loop being scheduled. + MachineLoop *getLoop() const { return Loop; } + + /// Return the number of stages contained in this schedule, which is the + /// largest stage index + 1. + int getNumStages() const { return NumStages; } + + /// Return the first cycle in the schedule, which is the cycle index of the + /// first instruction. + int getFirstCycle() { return Cycle[ScheduledInstrs.front()]; } + + /// Return the final cycle in the schedule, which is the cycle index of the + /// last instruction. + int getFinalCycle() { return Cycle[ScheduledInstrs.back()]; } + + /// Return the stage that MI is scheduled in, or -1. + int getStage(MachineInstr *MI) { + auto I = Stage.find(MI); + return I == Stage.end() ? -1 : I->second; + } + + /// Return the cycle that MI is scheduled at, or -1. + int getCycle(MachineInstr *MI) { + auto I = Cycle.find(MI); + return I == Cycle.end() ? -1 : I->second; + } + + /// Return the rescheduled instructions in order. + ArrayRef<MachineInstr *> getInstructions() { return ScheduledInstrs; } + + void dump() { print(dbgs()); } + void print(raw_ostream &OS); +}; + +/// The ModuloScheduleExpander takes a ModuloSchedule and expands it in-place, +/// rewriting the old loop and inserting prologs and epilogs as required. +class ModuloScheduleExpander { +public: + using InstrChangesTy = DenseMap<MachineInstr *, std::pair<unsigned, int64_t>>; + +private: + using ValueMapTy = DenseMap<unsigned, unsigned>; + using MBBVectorTy = SmallVectorImpl<MachineBasicBlock *>; + using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>; + + ModuloSchedule &Schedule; + MachineFunction &MF; + const TargetSubtargetInfo &ST; + MachineRegisterInfo &MRI; + const TargetInstrInfo *TII; + LiveIntervals &LIS; + + MachineBasicBlock *BB; + MachineBasicBlock *Preheader; + MachineBasicBlock *NewKernel = nullptr; + std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> LoopInfo; + + /// Map for each register and the max difference between its uses and def. + /// The first element in the pair is the max difference in stages. The + /// second is true if the register defines a Phi value and loop value is + /// scheduled before the Phi. + std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff; + + /// Instructions to change when emitting the final schedule. + InstrChangesTy InstrChanges; + + void generatePipelinedLoop(); + void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB, + ValueMapTy *VRMap, MBBVectorTy &PrologBBs); + void generateEpilog(unsigned LastStage, MachineBasicBlock *KernelBB, + ValueMapTy *VRMap, MBBVectorTy &EpilogBBs, + MBBVectorTy &PrologBBs); + void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + ValueMapTy *VRMap, InstrMapTy &InstrMap, + unsigned LastStageNum, unsigned CurStageNum, + bool IsLast); + void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + ValueMapTy *VRMap, InstrMapTy &InstrMap, + unsigned LastStageNum, unsigned CurStageNum, bool IsLast); + void removeDeadInstructions(MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs); + void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs); + void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs, + MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, + ValueMapTy *VRMap); + bool computeDelta(MachineInstr &MI, unsigned &Delta); + void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI, + unsigned Num); + MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum); + MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum); + void updateInstruction(MachineInstr *NewMI, bool LastDef, + unsigned CurStageNum, unsigned InstrStageNum, + ValueMapTy *VRMap); + MachineInstr *findDefInLoop(unsigned Reg); + unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal, + unsigned LoopStage, ValueMapTy *VRMap, + MachineBasicBlock *BB); + void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum, + ValueMapTy *VRMap, InstrMapTy &InstrMap); + void rewriteScheduledInstr(MachineBasicBlock *BB, InstrMapTy &InstrMap, + unsigned CurStageNum, unsigned PhiNum, + MachineInstr *Phi, unsigned OldReg, + unsigned NewReg, unsigned PrevReg = 0); + bool isLoopCarried(MachineInstr &Phi); + + /// Return the max. number of stages/iterations that can occur between a + /// register definition and its uses. + unsigned getStagesForReg(int Reg, unsigned CurStage) { + std::pair<unsigned, bool> Stages = RegToStageDiff[Reg]; + if ((int)CurStage > Schedule.getNumStages() - 1 && Stages.first == 0 && + Stages.second) + return 1; + return Stages.first; + } + + /// The number of stages for a Phi is a little different than other + /// instructions. The minimum value computed in RegToStageDiff is 1 + /// because we assume the Phi is needed for at least 1 iteration. + /// This is not the case if the loop value is scheduled prior to the + /// Phi in the same stage. This function returns the number of stages + /// or iterations needed between the Phi definition and any uses. + unsigned getStagesForPhi(int Reg) { + std::pair<unsigned, bool> Stages = RegToStageDiff[Reg]; + if (Stages.second) + return Stages.first; + return Stages.first - 1; + } + +public: + /// Create a new ModuloScheduleExpander. + /// \arg InstrChanges Modifications to make to instructions with memory + /// operands. + /// FIXME: InstrChanges is opaque and is an implementation detail of an + /// optimization in MachinePipeliner that crosses abstraction boundaries. + ModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S, + LiveIntervals &LIS, InstrChangesTy InstrChanges) + : Schedule(S), MF(MF), ST(MF.getSubtarget()), MRI(MF.getRegInfo()), + TII(ST.getInstrInfo()), LIS(LIS), + InstrChanges(std::move(InstrChanges)) {} + + /// Performs the actual expansion. + void expand(); + /// Performs final cleanup after expansion. + void cleanup(); + + /// Returns the newly rewritten kernel block, or nullptr if this was + /// optimized away. + MachineBasicBlock *getRewrittenKernel() { return NewKernel; } +}; + +/// A reimplementation of ModuloScheduleExpander. It works by generating a +/// standalone kernel loop and peeling out the prologs and epilogs. +class PeelingModuloScheduleExpander { + ModuloSchedule &Schedule; + MachineFunction &MF; + const TargetSubtargetInfo &ST; + MachineRegisterInfo &MRI; + const TargetInstrInfo *TII; + LiveIntervals *LIS; + + /// The original loop block that gets rewritten in-place. + MachineBasicBlock *BB; + /// The original loop preheader. + MachineBasicBlock *Preheader; + /// All prolog and epilog blocks. + SmallVector<MachineBasicBlock *, 4> Prologs, Epilogs; + /// For every block, the stages that are produced. + DenseMap<MachineBasicBlock *, BitVector> LiveStages; + /// For every block, the stages that are available. A stage can be available + /// but not produced (in the epilog) or produced but not available (in the + /// prolog). + DenseMap<MachineBasicBlock *, BitVector> AvailableStages; + + /// CanonicalMIs and BlockMIs form a bidirectional map between any of the + /// loop kernel clones. + DenseMap<MachineInstr *, MachineInstr *> CanonicalMIs; + DenseMap<std::pair<MachineBasicBlock *, MachineInstr *>, MachineInstr *> + BlockMIs; + + /// State passed from peelKernel to peelPrologAndEpilogs(). + std::deque<MachineBasicBlock *> PeeledFront, PeeledBack; + +public: + PeelingModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S, + LiveIntervals *LIS) + : Schedule(S), MF(MF), ST(MF.getSubtarget()), MRI(MF.getRegInfo()), + TII(ST.getInstrInfo()), LIS(LIS) {} + + void expand(); + + /// Runs ModuloScheduleExpander and treats it as a golden input to validate + /// aspects of the code generated by PeelingModuloScheduleExpander. + void validateAgainstModuloScheduleExpander(); + +protected: + /// Converts BB from the original loop body to the rewritten, pipelined + /// steady-state. + void rewriteKernel(); + +private: + /// Peels one iteration of the rewritten kernel (BB) in the specified + /// direction. + MachineBasicBlock *peelKernel(LoopPeelDirection LPD); + /// Peel the kernel forwards and backwards to produce prologs and epilogs, + /// and stitch them together. + void peelPrologAndEpilogs(); + /// All prolog and epilog blocks are clones of the kernel, so any produced + /// register in one block has an corollary in all other blocks. + Register getEquivalentRegisterIn(Register Reg, MachineBasicBlock *BB); + /// Change all users of MI, if MI is predicated out + /// (LiveStages[MI->getParent()] == false). + void rewriteUsesOf(MachineInstr *MI); + /// Insert branches between prologs, kernel and epilogs. + void fixupBranches(); + /// Create a poor-man's LCSSA by cloning only the PHIs from the kernel block + /// to a block dominated by all prologs and epilogs. This allows us to treat + /// the loop exiting block as any other kernel clone. + MachineBasicBlock *CreateLCSSAExitingBlock(); + /// Helper to get the stage of an instruction in the schedule. + unsigned getStage(MachineInstr *MI) { + if (CanonicalMIs.count(MI)) + MI = CanonicalMIs[MI]; + return Schedule.getStage(MI); + } +}; + +/// Expander that simply annotates each scheduled instruction with a post-instr +/// symbol that can be consumed by the ModuloScheduleTest pass. +/// +/// The post-instr symbol is a way of annotating an instruction that can be +/// roundtripped in MIR. The syntax is: +/// MYINST %0, post-instr-symbol <mcsymbol Stage-1_Cycle-5> +class ModuloScheduleTestAnnotater { + MachineFunction &MF; + ModuloSchedule &S; + +public: + ModuloScheduleTestAnnotater(MachineFunction &MF, ModuloSchedule &S) + : MF(MF), S(S) {} + + /// Performs the annotation. + void annotate(); +}; + +} // end namespace llvm + +#endif // LLVM_LIB_CODEGEN_MODULOSCHEDULE_H diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h index 8b014ccbb07b..099ba788e9a2 100644 --- a/include/llvm/CodeGen/PBQP/Math.h +++ b/include/llvm/CodeGen/PBQP/Math.h @@ -28,17 +28,17 @@ class Vector { public: /// Construct a PBQP vector of the given size. explicit Vector(unsigned Length) - : Length(Length), Data(llvm::make_unique<PBQPNum []>(Length)) {} + : Length(Length), Data(std::make_unique<PBQPNum []>(Length)) {} /// Construct a PBQP vector with initializer. Vector(unsigned Length, PBQPNum InitVal) - : Length(Length), Data(llvm::make_unique<PBQPNum []>(Length)) { + : Length(Length), Data(std::make_unique<PBQPNum []>(Length)) { std::fill(Data.get(), Data.get() + Length, InitVal); } /// Copy construct a PBQP vector. Vector(const Vector &V) - : Length(V.Length), Data(llvm::make_unique<PBQPNum []>(Length)) { + : Length(V.Length), Data(std::make_unique<PBQPNum []>(Length)) { std::copy(V.Data.get(), V.Data.get() + Length, Data.get()); } @@ -125,21 +125,21 @@ private: public: /// Construct a PBQP Matrix with the given dimensions. Matrix(unsigned Rows, unsigned Cols) : - Rows(Rows), Cols(Cols), Data(llvm::make_unique<PBQPNum []>(Rows * Cols)) { + Rows(Rows), Cols(Cols), Data(std::make_unique<PBQPNum []>(Rows * Cols)) { } /// Construct a PBQP Matrix with the given dimensions and initial /// value. Matrix(unsigned Rows, unsigned Cols, PBQPNum InitVal) : Rows(Rows), Cols(Cols), - Data(llvm::make_unique<PBQPNum []>(Rows * Cols)) { + Data(std::make_unique<PBQPNum []>(Rows * Cols)) { std::fill(Data.get(), Data.get() + (Rows * Cols), InitVal); } /// Copy construct a PBQP matrix. Matrix(const Matrix &M) : Rows(M.Rows), Cols(M.Cols), - Data(llvm::make_unique<PBQPNum []>(Rows * Cols)) { + Data(std::make_unique<PBQPNum []>(Rows * Cols)) { std::copy(M.Data.get(), M.Data.get() + (Rows * Cols), Data.get()); } diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index d92ee93268e7..1e765ce51e4a 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -226,6 +226,10 @@ namespace llvm { /// inserting cmov instructions. extern char &EarlyIfConverterID; + /// EarlyIfPredicator - This pass performs if-conversion on SSA form by + /// predicating if/else block and insert select at the join point. + extern char &EarlyIfPredicatorID; + /// This pass performs instruction combining using trace metrics to estimate /// critical-path and resource depth. extern char &MachineCombinerID; diff --git a/include/llvm/CodeGen/Register.h b/include/llvm/CodeGen/Register.h index 907c1a99e56f..aa5173684e24 100644 --- a/include/llvm/CodeGen/Register.h +++ b/include/llvm/CodeGen/Register.h @@ -9,6 +9,7 @@ #ifndef LLVM_CODEGEN_REGISTER_H #define LLVM_CODEGEN_REGISTER_H +#include "llvm/MC/MCRegister.h" #include <cassert> namespace llvm { @@ -20,41 +21,136 @@ class Register { public: Register(unsigned Val = 0): Reg(Val) {} + Register(MCRegister Val): Reg(Val) {} + + // Register numbers can represent physical registers, virtual registers, and + // sometimes stack slots. The unsigned values are divided into these ranges: + // + // 0 Not a register, can be used as a sentinel. + // [1;2^30) Physical registers assigned by TableGen. + // [2^30;2^31) Stack slots. (Rarely used.) + // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. + // + // Further sentinels can be allocated from the small negative integers. + // DenseMapInfo<unsigned> uses -1u and -2u. + + /// isStackSlot - Sometimes it is useful the be able to store a non-negative + /// frame index in a variable that normally holds a register. isStackSlot() + /// returns true if Reg is in the range used for stack slots. + /// + /// Note that isVirtualRegister() and isPhysicalRegister() cannot handle stack + /// slots, so if a variable may contains a stack slot, always check + /// isStackSlot() first. + /// + static bool isStackSlot(unsigned Reg) { + return MCRegister::isStackSlot(Reg); + } + + /// Compute the frame index from a register value representing a stack slot. + static int stackSlot2Index(unsigned Reg) { + assert(isStackSlot(Reg) && "Not a stack slot"); + return int(Reg - (1u << 30)); + } + + /// Convert a non-negative frame index to a stack slot register value. + static unsigned index2StackSlot(int FI) { + assert(FI >= 0 && "Cannot hold a negative frame index."); + return FI + (1u << 30); + } + + /// Return true if the specified register number is in + /// the physical register namespace. + static bool isPhysicalRegister(unsigned Reg) { + return MCRegister::isPhysicalRegister(Reg); + } + + /// Return true if the specified register number is in + /// the virtual register namespace. + static bool isVirtualRegister(unsigned Reg) { + assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); + return int(Reg) < 0; + } + + /// Convert a virtual register number to a 0-based index. + /// The first virtual register in a function will get the index 0. + static unsigned virtReg2Index(unsigned Reg) { + assert(isVirtualRegister(Reg) && "Not a virtual register"); + return Reg & ~(1u << 31); + } + + /// Convert a 0-based index to a virtual register number. + /// This is the inverse operation of VirtReg2IndexFunctor below. + static unsigned index2VirtReg(unsigned Index) { + return Index | (1u << 31); + } /// Return true if the specified register number is in the virtual register /// namespace. bool isVirtual() const { - return int(Reg) < 0; + return isVirtualRegister(Reg); } /// Return true if the specified register number is in the physical register /// namespace. bool isPhysical() const { - return int(Reg) > 0; + return isPhysicalRegister(Reg); } /// Convert a virtual register number to a 0-based index. The first virtual /// register in a function will get the index 0. unsigned virtRegIndex() const { - assert(isVirtual() && "Not a virtual register"); - return Reg & ~(1u << 31); - } - - /// Convert a 0-based index to a virtual register number. - /// This is the inverse operation of VirtReg2IndexFunctor below. - static Register index2VirtReg(unsigned Index) { - return Register(Index | (1u << 31)); + return virtReg2Index(Reg); } operator unsigned() const { return Reg; } + unsigned id() const { return Reg; } + + operator MCRegister() const { + return MCRegister(Reg); + } + bool isValid() const { return Reg != 0; } + + /// Comparisons between register objects + bool operator==(const Register &Other) const { return Reg == Other.Reg; } + bool operator!=(const Register &Other) const { return Reg != Other.Reg; } + bool operator==(const MCRegister &Other) const { return Reg == Other.id(); } + bool operator!=(const MCRegister &Other) const { return Reg != Other.id(); } + + /// Comparisons against register constants. E.g. + /// * R == AArch64::WZR + /// * R == 0 + /// * R == VirtRegMap::NO_PHYS_REG + bool operator==(unsigned Other) const { return Reg == Other; } + bool operator!=(unsigned Other) const { return Reg != Other; } + bool operator==(int Other) const { return Reg == unsigned(Other); } + bool operator!=(int Other) const { return Reg != unsigned(Other); } + // MSVC requires that we explicitly declare these two as well. + bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } + bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } +}; + +// Provide DenseMapInfo for Register +template<> struct DenseMapInfo<Register> { + static inline unsigned getEmptyKey() { + return DenseMapInfo<unsigned>::getEmptyKey(); + } + static inline unsigned getTombstoneKey() { + return DenseMapInfo<unsigned>::getTombstoneKey(); + } + static unsigned getHashValue(const Register &Val) { + return DenseMapInfo<unsigned>::getHashValue(Val.id()); + } + static bool isEqual(const Register &LHS, const Register &RHS) { + return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); + } }; } -#endif +#endif // ifndef LLVM_CODEGEN_REGISTER_H diff --git a/include/llvm/CodeGen/RegisterClassInfo.h b/include/llvm/CodeGen/RegisterClassInfo.h index 14af5c4d090d..25b310c47621 100644 --- a/include/llvm/CodeGen/RegisterClassInfo.h +++ b/include/llvm/CodeGen/RegisterClassInfo.h @@ -110,7 +110,7 @@ public: /// getLastCalleeSavedAlias - Returns the last callee saved register that /// overlaps PhysReg, or 0 if Reg doesn't overlap a CalleeSavedAliases. unsigned getLastCalleeSavedAlias(unsigned PhysReg) const { - assert(TargetRegisterInfo::isPhysicalRegister(PhysReg)); + assert(Register::isPhysicalRegister(PhysReg)); if (PhysReg < CalleeSavedAliases.size()) return CalleeSavedAliases[PhysReg]; return 0; diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index 5bbaa03fd751..92333b859f1b 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -129,6 +129,8 @@ public: bool operator==(const PressureChange &RHS) const { return PSetID == RHS.PSetID && UnitInc == RHS.UnitInc; } + + void dump() const; }; /// List of PressureChanges in order of increasing, unique PSetID. @@ -248,6 +250,7 @@ struct RegPressureDelta { bool operator!=(const RegPressureDelta &RHS) const { return !operator==(RHS); } + void dump() const; }; /// A set of live virtual registers and physical register units. @@ -273,15 +276,15 @@ private: unsigned NumRegUnits; unsigned getSparseIndexFromReg(unsigned Reg) const { - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return TargetRegisterInfo::virtReg2Index(Reg) + NumRegUnits; + if (Register::isVirtualRegister(Reg)) + return Register::virtReg2Index(Reg) + NumRegUnits; assert(Reg < NumRegUnits); return Reg; } unsigned getRegFromSparseIndex(unsigned SparseIndex) const { if (SparseIndex >= NumRegUnits) - return TargetRegisterInfo::index2VirtReg(SparseIndex-NumRegUnits); + return Register::index2VirtReg(SparseIndex-NumRegUnits); return SparseIndex; } diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h index 9c48df82f07d..5b5a80a67e7f 100644 --- a/include/llvm/CodeGen/RegisterScavenging.h +++ b/include/llvm/CodeGen/RegisterScavenging.h @@ -51,7 +51,7 @@ class RegScavenger { /// If non-zero, the specific register is currently being /// scavenged. That is, it is spilled to this scavenging stack slot. - unsigned Reg = 0; + Register Reg; /// The instruction that restores the scavenged register from stack. const MachineInstr *Restore = nullptr; @@ -119,14 +119,14 @@ public: MachineBasicBlock::iterator getCurrentPosition() const { return MBBI; } /// Return if a specific register is currently used. - bool isRegUsed(unsigned Reg, bool includeReserved = true) const; + bool isRegUsed(Register Reg, bool includeReserved = true) const; /// Return all available registers in the register class in Mask. BitVector getRegsAvailable(const TargetRegisterClass *RC); /// Find an unused register of the specified register class. /// Return 0 if none is found. - unsigned FindUnusedReg(const TargetRegisterClass *RC) const; + Register FindUnusedReg(const TargetRegisterClass *RC) const; /// Add a scavenging frame index. void addScavengingFrameIndex(int FI) { @@ -160,10 +160,10 @@ public: /// /// If \p AllowSpill is false, fail if a spill is required to make the /// register available, and return NoRegister. - unsigned scavengeRegister(const TargetRegisterClass *RC, + Register scavengeRegister(const TargetRegisterClass *RC, MachineBasicBlock::iterator I, int SPAdj, bool AllowSpill = true); - unsigned scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj, + Register scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj, bool AllowSpill = true) { return scavengeRegister(RegClass, MBBI, SPAdj, AllowSpill); } @@ -177,17 +177,17 @@ public: /// /// If \p AllowSpill is false, fail if a spill is required to make the /// register available, and return NoRegister. - unsigned scavengeRegisterBackwards(const TargetRegisterClass &RC, + Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill = true); /// Tell the scavenger a register is used. - void setRegUsed(unsigned Reg, LaneBitmask LaneMask = LaneBitmask::getAll()); + void setRegUsed(Register Reg, LaneBitmask LaneMask = LaneBitmask::getAll()); private: /// Returns true if a register is reserved. It is never "unused". - bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); } + bool isReserved(Register Reg) const { return MRI->isReserved(Reg); } /// setUsed / setUnused - Mark the state of one or a number of register units. /// @@ -203,16 +203,16 @@ private: void determineKillsAndDefs(); /// Add all Reg Units that Reg contains to BV. - void addRegUnits(BitVector &BV, unsigned Reg); + void addRegUnits(BitVector &BV, Register Reg); /// Remove all Reg Units that \p Reg contains from \p BV. - void removeRegUnits(BitVector &BV, unsigned Reg); + void removeRegUnits(BitVector &BV, Register Reg); /// Return the candidate register that is unused for the longest after /// StartMI. UseMI is set to the instruction where the search stopped. /// /// No more than InstrLimit instructions are inspected. - unsigned findSurvivorReg(MachineBasicBlock::iterator StartMI, + Register findSurvivorReg(MachineBasicBlock::iterator StartMI, BitVector &Candidates, unsigned InstrLimit, MachineBasicBlock::iterator &UseMI); @@ -225,7 +225,7 @@ private: /// Spill a register after position \p After and reload it before position /// \p UseMI. - ScavengedInfo &spill(unsigned Reg, const TargetRegisterClass &RC, int SPAdj, + ScavengedInfo &spill(Register Reg, const TargetRegisterClass &RC, int SPAdj, MachineBasicBlock::iterator Before, MachineBasicBlock::iterator &UseMI); }; diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index 3e3b604acbac..1eb9b9f322ba 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -34,6 +34,7 @@ namespace llvm { + class AAResults; class LiveIntervals; class MachineFrameInfo; class MachineFunction; @@ -57,7 +58,7 @@ namespace llvm { : VirtReg(VReg), LaneMask(LaneMask), SU(SU) {} unsigned getSparseSetIndex() const { - return TargetRegisterInfo::virtReg2Index(VirtReg); + return Register::virtReg2Index(VirtReg); } }; @@ -173,7 +174,7 @@ namespace llvm { /// Tracks the last instructions in this region using each virtual register. VReg2SUnitOperIdxMultiMap CurrentVRegUses; - AliasAnalysis *AAForDep = nullptr; + AAResults *AAForDep = nullptr; /// Remember a generic side-effecting instruction as we proceed. /// No other SU ever gets scheduled around it (except in the special @@ -201,7 +202,7 @@ namespace llvm { Value2SUsMap &loads, unsigned N); /// Adds a chain edge between SUa and SUb, but only if both - /// AliasAnalysis and Target fail to deny the dependency. + /// AAResults and Target fail to deny the dependency. void addChainDependency(SUnit *SUa, SUnit *SUb, unsigned Latency = 0); @@ -306,7 +307,7 @@ namespace llvm { /// If \p RPTracker is non-null, compute register pressure as a side effect. /// The DAG builder is an efficient place to do it because it already visits /// operands. - void buildSchedGraph(AliasAnalysis *AA, + void buildSchedGraph(AAResults *AA, RegPressureTracker *RPTracker = nullptr, PressureDiffs *PDiffs = nullptr, LiveIntervals *LIS = nullptr, @@ -374,6 +375,9 @@ namespace llvm { /// Returns a mask for which lanes get read/written by the given (register) /// machine operand. LaneBitmask getLaneMaskForMO(const MachineOperand &MO) const; + + /// Returns true if the def register in \p MO has no uses. + bool deadDefHasNoUse(const MachineOperand &MO); }; /// Creates a new SUnit and return a ptr to it. diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 12a970847021..6b8e2dd803ba 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -26,8 +26,6 @@ #include "llvm/ADT/ilist.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" @@ -58,6 +56,7 @@ namespace llvm { +class AAResults; class BlockAddress; class Constant; class ConstantFP; @@ -66,6 +65,7 @@ class DataLayout; struct fltSemantics; class GlobalValue; struct KnownBits; +class LegacyDivergenceAnalysis; class LLVMContext; class MachineBasicBlock; class MachineConstantPoolValue; @@ -269,7 +269,13 @@ class SelectionDAG { using CallSiteInfo = MachineFunction::CallSiteInfo; using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; - DenseMap<const SDNode *, CallSiteInfo> SDCallSiteInfo; + + struct CallSiteDbgInfo { + CallSiteInfo CSInfo; + MDNode *HeapAllocSite = nullptr; + }; + + DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo; uint16_t NextPersistentId = 0; @@ -382,7 +388,11 @@ private: Node->OperandList = nullptr; } void CreateTopologicalOrder(std::vector<SDNode*>& Order); + public: + // Maximum depth for recursive analysis such as computeKnownBits, etc. + static constexpr unsigned MaxRecursionDepth = 6; + explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); SelectionDAG(const SelectionDAG &) = delete; SelectionDAG &operator=(const SelectionDAG &) = delete; @@ -489,7 +499,7 @@ public: /// certain types of nodes together, or eliminating superfluous nodes. The /// Level argument controls whether Combine is allowed to produce nodes and /// types that are illegal on the target. - void Combine(CombineLevel Level, AliasAnalysis *AA, + void Combine(CombineLevel Level, AAResults *AA, CodeGenOpt::Level OptLevel); /// This transforms the SelectionDAG into a SelectionDAG that @@ -628,10 +638,9 @@ public: SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset = 0, bool isTargetGA = false, - unsigned char TargetFlags = 0); + unsigned TargetFlags = 0); SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, - int64_t offset = 0, - unsigned char TargetFlags = 0) { + int64_t offset = 0, unsigned TargetFlags = 0) { return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); } SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); @@ -639,28 +648,27 @@ public: return getFrameIndex(FI, VT, true); } SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false, - unsigned char TargetFlags = 0); - SDValue getTargetJumpTable(int JTI, EVT VT, unsigned char TargetFlags = 0) { + unsigned TargetFlags = 0); + SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) { return getJumpTable(JTI, VT, true, TargetFlags); } - SDValue getConstantPool(const Constant *C, EVT VT, - unsigned Align = 0, int Offs = 0, bool isT=false, - unsigned char TargetFlags = 0); - SDValue getTargetConstantPool(const Constant *C, EVT VT, - unsigned Align = 0, int Offset = 0, - unsigned char TargetFlags = 0) { + SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align = 0, + int Offs = 0, bool isT = false, + unsigned TargetFlags = 0); + SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align = 0, + int Offset = 0, unsigned TargetFlags = 0) { return getConstantPool(C, VT, Align, Offset, true, TargetFlags); } SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT, unsigned Align = 0, int Offs = 0, bool isT=false, - unsigned char TargetFlags = 0); - SDValue getTargetConstantPool(MachineConstantPoolValue *C, - EVT VT, unsigned Align = 0, - int Offset = 0, unsigned char TargetFlags=0) { + unsigned TargetFlags = 0); + SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, + unsigned Align = 0, int Offset = 0, + unsigned TargetFlags = 0) { return getConstantPool(C, VT, Align, Offset, true, TargetFlags); } SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0, - unsigned char TargetFlags = 0); + unsigned TargetFlags = 0); // When generating a branch to a BB, we don't in general know enough // to provide debug info for the BB at that time, so keep this one around. SDValue getBasicBlock(MachineBasicBlock *MBB); @@ -668,7 +676,7 @@ public: SDValue getExternalSymbol(const char *Sym, EVT VT); SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT); SDValue getTargetExternalSymbol(const char *Sym, EVT VT, - unsigned char TargetFlags = 0); + unsigned TargetFlags = 0); SDValue getMCSymbol(MCSymbol *Sym, EVT VT); SDValue getValueType(EVT); @@ -677,12 +685,10 @@ public: SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label); - SDValue getBlockAddress(const BlockAddress *BA, EVT VT, - int64_t Offset = 0, bool isTarget = false, - unsigned char TargetFlags = 0); + SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, + bool isTarget = false, unsigned TargetFlags = 0); SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, - int64_t Offset = 0, - unsigned char TargetFlags = 0) { + int64_t Offset = 0, unsigned TargetFlags = 0) { return getBlockAddress(BA, VT, Offset, true, TargetFlags); } @@ -1035,7 +1041,7 @@ public: unsigned Align = 0, MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore, - unsigned Size = 0, + uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, @@ -1117,9 +1123,11 @@ public: MachineMemOperand *MMO, bool IsTruncating = false, bool IsCompressing = false); SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, - ArrayRef<SDValue> Ops, MachineMemOperand *MMO); + ArrayRef<SDValue> Ops, MachineMemOperand *MMO, + ISD::MemIndexType IndexType); SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, - ArrayRef<SDValue> Ops, MachineMemOperand *MMO); + ArrayRef<SDValue> Ops, MachineMemOperand *MMO, + ISD::MemIndexType IndexType); /// Return (create a new or find existing) a target-specific node. /// TargetMemSDNode should be derived class from MemSDNode. @@ -1588,9 +1596,12 @@ public: /// Extract. The reduction must use one of the opcodes listed in /p /// CandidateBinOps and on success /p BinOp will contain the matching opcode. /// Returns the vector that is being reduced on, or SDValue() if a reduction - /// was not matched. + /// was not matched. If \p AllowPartials is set then in the case of a + /// reduction pattern that only matches the first few stages, the extracted + /// subvector of the start of the reduction is returned. SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, - ArrayRef<ISD::NodeType> CandidateBinOps); + ArrayRef<ISD::NodeType> CandidateBinOps, + bool AllowPartials = false); /// Utility function used by legalize and lowering to /// "unroll" a vector operation by splitting out the scalars and operating @@ -1664,16 +1675,28 @@ public: } void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { - SDCallSiteInfo[CallNode] = std::move(CallInfo); + SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo); } CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) { - auto I = SDCallSiteInfo.find(CallNode); - if (I != SDCallSiteInfo.end()) - return std::move(I->second); + auto I = SDCallSiteDbgInfo.find(CallNode); + if (I != SDCallSiteDbgInfo.end()) + return std::move(I->second).CSInfo; return CallSiteInfo(); } + void addHeapAllocSite(const SDNode *Node, MDNode *MD) { + SDCallSiteDbgInfo[Node].HeapAllocSite = MD; + } + + /// Return the HeapAllocSite type associated with the SDNode, if it exists. + MDNode *getHeapAllocSite(const SDNode *Node) { + auto It = SDCallSiteDbgInfo.find(Node); + if (It == SDCallSiteDbgInfo.end()) + return nullptr; + return It->second.HeapAllocSite; + } + private: void InsertNode(SDNode *N); bool RemoveNodeFromCSEMaps(SDNode *N); @@ -1712,7 +1735,7 @@ private: std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes; StringMap<SDNode*> ExternalSymbols; - std::map<std::pair<std::string, unsigned char>,SDNode*> TargetExternalSymbols; + std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols; DenseMap<MCSymbol *, SDNode *> MCSymbols; }; diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index 147c325342fc..de71a21d4671 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -22,22 +22,23 @@ #include <memory> namespace llvm { - class FastISel; - class SelectionDAGBuilder; - class SDValue; - class MachineRegisterInfo; - class MachineBasicBlock; - class MachineFunction; - class MachineInstr; - class OptimizationRemarkEmitter; - class TargetLowering; - class TargetLibraryInfo; - class FunctionLoweringInfo; - class ScheduleHazardRecognizer; - class SwiftErrorValueTracking; - class GCFunctionInfo; - class ScheduleDAGSDNodes; - class LoadInst; +class AAResults; +class FastISel; +class SelectionDAGBuilder; +class SDValue; +class MachineRegisterInfo; +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class OptimizationRemarkEmitter; +class TargetLowering; +class TargetLibraryInfo; +class FunctionLoweringInfo; +class ScheduleHazardRecognizer; +class SwiftErrorValueTracking; +class GCFunctionInfo; +class ScheduleDAGSDNodes; +class LoadInst; /// SelectionDAGISel - This is the common base class used for SelectionDAG-based /// pattern-matching instruction selectors. @@ -51,7 +52,7 @@ public: MachineRegisterInfo *RegInfo; SelectionDAG *CurDAG; SelectionDAGBuilder *SDB; - AliasAnalysis *AA; + AAResults *AA; GCFunctionInfo *GFI; CodeGenOpt::Level OptLevel; const TargetInstrInfo *TII; @@ -162,6 +163,7 @@ public: OPC_EmitMergeInputChains1_1, OPC_EmitMergeInputChains1_2, OPC_EmitCopyToReg, + OPC_EmitCopyToReg2, OPC_EmitNodeXForm, OPC_EmitNode, // Space-optimized forms that implicitly encode number of result VTs. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 5aab9643e09d..ceb8b72635a2 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -548,10 +548,15 @@ BEGIN_TWO_BYTE_PACK() class LSBaseSDNodeBitfields { friend class LSBaseSDNode; + friend class MaskedGatherScatterSDNode; uint16_t : NumMemSDNodeBits; - uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode + // This storage is shared between disparate class hierarchies to hold an + // enumeration specific to the class hierarchy in use. + // LSBaseSDNode => enum ISD::MemIndexedMode + // MaskedGatherScatterSDNode => enum ISD::MemIndexType + uint16_t AddressingMode : 3; }; enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; @@ -696,14 +701,20 @@ public: case ISD::STRICT_FLOG: case ISD::STRICT_FLOG10: case ISD::STRICT_FLOG2: + case ISD::STRICT_LRINT: + case ISD::STRICT_LLRINT: case ISD::STRICT_FRINT: case ISD::STRICT_FNEARBYINT: case ISD::STRICT_FMAXNUM: case ISD::STRICT_FMINNUM: case ISD::STRICT_FCEIL: case ISD::STRICT_FFLOOR: + case ISD::STRICT_LROUND: + case ISD::STRICT_LLROUND: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_TO_SINT: + case ISD::STRICT_FP_TO_UINT: case ISD::STRICT_FP_ROUND: case ISD::STRICT_FP_EXTEND: return true; @@ -1346,6 +1357,17 @@ public: /// store occurs. AtomicOrdering getOrdering() const { return MMO->getOrdering(); } + /// Return true if the memory operation ordering is Unordered or higher. + bool isAtomic() const { return MMO->isAtomic(); } + + /// Returns true if the memory operation doesn't imply any ordering + /// constraints on surrounding memory operations beyond the normal memory + /// aliasing rules. + bool isUnordered() const { return MMO->isUnordered(); } + + /// Returns true if the memory operation is neither atomic or volatile. + bool isSimple() const { return !isAtomic() && !isVolatile(); } + /// Return the type of the in-memory value. EVT getMemoryVT() const { return MemoryVT; } @@ -1702,16 +1724,16 @@ class GlobalAddressSDNode : public SDNode { const GlobalValue *TheGlobal; int64_t Offset; - unsigned char TargetFlags; + unsigned TargetFlags; GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, const GlobalValue *GA, EVT VT, int64_t o, - unsigned char TF); + unsigned TF); public: const GlobalValue *getGlobal() const { return TheGlobal; } int64_t getOffset() const { return Offset; } - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } // Return the address space this GlobalAddress belongs to. unsigned getAddressSpace() const; @@ -1778,16 +1800,16 @@ class JumpTableSDNode : public SDNode { friend class SelectionDAG; int JTI; - unsigned char TargetFlags; + unsigned TargetFlags; - JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF) + JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { } public: int getIndex() const { return JTI; } - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::JumpTable || @@ -1804,10 +1826,10 @@ class ConstantPoolSDNode : public SDNode { } Val; int Offset; // It's a MachineConstantPoolValue if top bit is set. unsigned Alignment; // Minimum alignment requirement of CP (not log2 value). - unsigned char TargetFlags; + unsigned TargetFlags; ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, - unsigned Align, unsigned char TF) + unsigned Align, unsigned TF) : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), TargetFlags(TF) { @@ -1816,7 +1838,7 @@ class ConstantPoolSDNode : public SDNode { } ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, - EVT VT, int o, unsigned Align, unsigned char TF) + EVT VT, int o, unsigned Align, unsigned TF) : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), TargetFlags(TF) { @@ -1847,7 +1869,7 @@ public: // Return the alignment of this constant pool object, which is either 0 (for // default alignment) or the desired value. unsigned getAlignment() const { return Alignment; } - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } Type *getType() const; @@ -1861,16 +1883,16 @@ public: class TargetIndexSDNode : public SDNode { friend class SelectionDAG; - unsigned char TargetFlags; + unsigned TargetFlags; int Index; int64_t Offset; public: - TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF) - : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), - TargetFlags(TF), Index(Idx), Offset(Ofs) {} + TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) + : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), + TargetFlags(TF), Index(Idx), Offset(Ofs) {} - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } int getIndex() const { return Index; } int64_t getOffset() const { return Offset; } @@ -2063,17 +2085,17 @@ class BlockAddressSDNode : public SDNode { const BlockAddress *BA; int64_t Offset; - unsigned char TargetFlags; + unsigned TargetFlags; BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, - int64_t o, unsigned char Flags) + int64_t o, unsigned Flags) : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), BA(ba), Offset(o), TargetFlags(Flags) {} public: const BlockAddress *getBlockAddress() const { return BA; } int64_t getOffset() const { return Offset; } - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::BlockAddress || @@ -2104,15 +2126,16 @@ class ExternalSymbolSDNode : public SDNode { friend class SelectionDAG; const char *Symbol; - unsigned char TargetFlags; + unsigned TargetFlags; - ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned char TF, EVT VT) - : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, - 0, DebugLoc(), getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) {} + ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) + : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, + DebugLoc(), getSDVTList(VT)), + Symbol(Sym), TargetFlags(TF) {} public: const char *getSymbol() const { return Symbol; } - unsigned char getTargetFlags() const { return TargetFlags; } + unsigned getTargetFlags() const { return TargetFlags; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ExternalSymbol || @@ -2181,8 +2204,6 @@ public: : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { LSBaseSDNodeBits.AddressingMode = AM; assert(getAddressingMode() == AM && "Value truncated"); - assert((!MMO->isAtomic() || MMO->isVolatile()) && - "use an AtomicSDNode instead for non-volatile atomics"); } const SDValue &getOffset() const { @@ -2362,8 +2383,24 @@ public: MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, - MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} + MachineMemOperand *MMO, ISD::MemIndexType IndexType) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + LSBaseSDNodeBits.AddressingMode = IndexType; + assert(getIndexType() == IndexType && "Value truncated"); + } + + /// How is Index applied to BasePtr when computing addresses. + ISD::MemIndexType getIndexType() const { + return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); + } + bool isIndexScaled() const { + return (getIndexType() == ISD::SIGNED_SCALED) || + (getIndexType() == ISD::UNSIGNED_SCALED); + } + bool isIndexSigned() const { + return (getIndexType() == ISD::SIGNED_SCALED) || + (getIndexType() == ISD::SIGNED_UNSCALED); + } // In the both nodes address is Op1, mask is Op2: // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) @@ -2387,8 +2424,10 @@ public: friend class SelectionDAG; MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - EVT MemVT, MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} + EVT MemVT, MachineMemOperand *MMO, + ISD::MemIndexType IndexType) + : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, + IndexType) {} const SDValue &getPassThru() const { return getOperand(1); } @@ -2404,8 +2443,10 @@ public: friend class SelectionDAG; MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - EVT MemVT, MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} + EVT MemVT, MachineMemOperand *MMO, + ISD::MemIndexType IndexType) + : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, + IndexType) {} const SDValue &getValue() const { return getOperand(1); } diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h index 2bdf4425e24a..ed52db3e6269 100644 --- a/include/llvm/CodeGen/StackProtector.h +++ b/include/llvm/CodeGen/StackProtector.h @@ -61,6 +61,12 @@ private: /// protection when -fstack-protection is used. unsigned SSPBufferSize = 0; + /// VisitedPHIs - The set of PHI nodes visited when determining + /// if a variable's reference has been taken. This set + /// is maintained to ensure we don't visit the same PHI node multiple + /// times. + SmallPtrSet<const PHINode *, 16> VisitedPHIs; + // A prologue is generated. bool HasPrologue = false; diff --git a/include/llvm/CodeGen/SwitchLoweringUtils.h b/include/llvm/CodeGen/SwitchLoweringUtils.h index 62134dc792f7..b8adcf759b19 100644 --- a/include/llvm/CodeGen/SwitchLoweringUtils.h +++ b/include/llvm/CodeGen/SwitchLoweringUtils.h @@ -212,16 +212,17 @@ struct BitTestBlock { BitTestInfo Cases; BranchProbability Prob; BranchProbability DefaultProb; + bool OmitRangeCheck; BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, bool E, bool CR, MachineBasicBlock *P, MachineBasicBlock *D, BitTestInfo C, BranchProbability Pr) : First(std::move(F)), Range(std::move(R)), SValue(SV), Reg(Rg), RegVT(RgVT), Emitted(E), ContiguousRange(CR), Parent(P), Default(D), - Cases(std::move(C)), Prob(Pr) {} + Cases(std::move(C)), Prob(Pr), OmitRangeCheck(false) {} }; -/// Return the range of value within a range. +/// Return the range of values within a range. uint64_t getJumpTableRange(const CaseClusterVector &Clusters, unsigned First, unsigned Last); diff --git a/include/llvm/CodeGen/TargetCallingConv.h b/include/llvm/CodeGen/TargetCallingConv.h index aebeeecbe506..db3d1175afee 100644 --- a/include/llvm/CodeGen/TargetCallingConv.h +++ b/include/llvm/CodeGen/TargetCallingConv.h @@ -14,6 +14,7 @@ #define LLVM_CODEGEN_TARGETCALLINGCONV_H #include "llvm/CodeGen/ValueTypes.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include <cassert> @@ -120,16 +121,22 @@ namespace ISD { bool isPointer() const { return IsPointer; } void setPointer() { IsPointer = 1; } - unsigned getByValAlign() const { return (1U << ByValAlign) / 2; } - void setByValAlign(unsigned A) { - ByValAlign = Log2_32(A) + 1; - assert(getByValAlign() == A && "bitfield overflow"); + unsigned getByValAlign() const { + MaybeAlign A = decodeMaybeAlign(ByValAlign); + return A ? A->value() : 0; + } + void setByValAlign(Align A) { + ByValAlign = encode(A); + assert(getByValAlign() == A.value() && "bitfield overflow"); } - unsigned getOrigAlign() const { return (1U << OrigAlign) / 2; } - void setOrigAlign(unsigned A) { - OrigAlign = Log2_32(A) + 1; - assert(getOrigAlign() == A && "bitfield overflow"); + unsigned getOrigAlign() const { + MaybeAlign A = decodeMaybeAlign(OrigAlign); + return A ? A->value() : 0; + } + void setOrigAlign(Align A) { + OrigAlign = encode(A); + assert(getOrigAlign() == A.value() && "bitfield overflow"); } unsigned getByValSize() const { return ByValSize; } diff --git a/include/llvm/CodeGen/TargetFrameLowering.h b/include/llvm/CodeGen/TargetFrameLowering.h index 878c9ffd2b51..72edb27964c4 100644 --- a/include/llvm/CodeGen/TargetFrameLowering.h +++ b/include/llvm/CodeGen/TargetFrameLowering.h @@ -28,6 +28,7 @@ namespace TargetStackID { enum Value { Default = 0, SGPRSpill = 1, + SVEVector = 2, NoAlloc = 255 }; } @@ -53,15 +54,15 @@ public: }; private: StackDirection StackDir; - unsigned StackAlignment; - unsigned TransientStackAlignment; + Align StackAlignment; + Align TransientStackAlignment; int LocalAreaOffset; bool StackRealignable; public: - TargetFrameLowering(StackDirection D, unsigned StackAl, int LAO, - unsigned TransAl = 1, bool StackReal = true) - : StackDir(D), StackAlignment(StackAl), TransientStackAlignment(TransAl), - LocalAreaOffset(LAO), StackRealignable(StackReal) {} + TargetFrameLowering(StackDirection D, Align StackAl, int LAO, + Align TransAl = Align::None(), bool StackReal = true) + : StackDir(D), StackAlignment(StackAl), TransientStackAlignment(TransAl), + LocalAreaOffset(LAO), StackRealignable(StackReal) {} virtual ~TargetFrameLowering(); @@ -76,7 +77,7 @@ public: /// stack pointer must be aligned on entry to a function. Typically, this /// is the largest alignment for any data object in the target. /// - unsigned getStackAlignment() const { return StackAlignment; } + unsigned getStackAlignment() const { return StackAlignment.value(); } /// alignSPAdjust - This method aligns the stack adjustment to the correct /// alignment. @@ -95,7 +96,7 @@ public: /// calls. /// unsigned getTransientStackAlignment() const { - return TransientStackAlignment; + return TransientStackAlignment.value(); } /// isStackRealignable - This method returns whether the stack can be @@ -366,15 +367,10 @@ public: /// Check if given function is safe for not having callee saved registers. /// This is used when interprocedural register allocation is enabled. - static bool isSafeForNoCSROpt(const Function &F) { - if (!F.hasLocalLinkage() || F.hasAddressTaken() || - !F.hasFnAttribute(Attribute::NoRecurse)) - return false; - // Function should not be optimized as tail call. - for (const User *U : F.users()) - if (auto CS = ImmutableCallSite(U)) - if (CS.isTailCall()) - return false; + static bool isSafeForNoCSROpt(const Function &F); + + /// Check if the no-CSR optimisation is profitable for the given function. + virtual bool isProfitableForNoCSROpt(const Function &F) const { return true; } diff --git a/include/llvm/CodeGen/TargetInstrInfo.h b/include/llvm/CodeGen/TargetInstrInfo.h index 25b04f8c019a..5011cf34c0ee 100644 --- a/include/llvm/CodeGen/TargetInstrInfo.h +++ b/include/llvm/CodeGen/TargetInstrInfo.h @@ -22,7 +22,7 @@ #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineOutliner.h" #include "llvm/CodeGen/PseudoSourceValue.h" @@ -38,10 +38,12 @@ namespace llvm { +class AAResults; class DFAPacketizer; class InstrItineraryData; class LiveIntervals; class LiveVariables; +class MachineLoop; class MachineMemOperand; class MachineRegisterInfo; class MCAsmInfo; @@ -60,6 +62,8 @@ class TargetSubtargetInfo; template <class T> class SmallVectorImpl; +using ParamLoadedValue = std::pair<MachineOperand, DIExpression*>; + //--------------------------------------------------------------------------- /// /// TargetInstrInfo - Interface to description of machine instruction set @@ -92,7 +96,7 @@ public: /// registers so that the instructions result is independent of the place /// in the function. bool isTriviallyReMaterializable(const MachineInstr &MI, - AliasAnalysis *AA = nullptr) const { + AAResults *AA = nullptr) const { return MI.getOpcode() == TargetOpcode::IMPLICIT_DEF || (MI.getDesc().isRematerializable() && (isReallyTriviallyReMaterializable(MI, AA) || @@ -108,7 +112,7 @@ protected: /// not always available. /// Requirements must be check as stated in isTriviallyReMaterializable() . virtual bool isReallyTriviallyReMaterializable(const MachineInstr &MI, - AliasAnalysis *AA) const { + AAResults *AA) const { return false; } @@ -151,7 +155,7 @@ private: /// this function does target-independent tests to determine if the /// instruction is really trivially rematerializable. bool isReallyTriviallyReMaterializableGeneric(const MachineInstr &MI, - AliasAnalysis *AA) const; + AAResults *AA) const; public: /// These methods return the opcode of the frame setup/destroy instructions @@ -419,7 +423,8 @@ public: /// findCommutedOpIndices(MI, Op1, Op2); /// can be interpreted as a query asking to find an operand that would be /// commutable with the operand#1. - virtual bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1, + virtual bool findCommutedOpIndices(const MachineInstr &MI, + unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const; /// A pair composed of a register and a sub-register index. @@ -659,6 +664,50 @@ public: BytesAdded); } + /// Object returned by analyzeLoopForPipelining. Allows software pipelining + /// implementations to query attributes of the loop being pipelined and to + /// apply target-specific updates to the loop once pipelining is complete. + class PipelinerLoopInfo { + public: + virtual ~PipelinerLoopInfo(); + /// Return true if the given instruction should not be pipelined and should + /// be ignored. An example could be a loop comparison, or induction variable + /// update with no users being pipelined. + virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const = 0; + + /// Create a condition to determine if the trip count of the loop is greater + /// than TC. + /// + /// If the trip count is statically known to be greater than TC, return + /// true. If the trip count is statically known to be not greater than TC, + /// return false. Otherwise return nullopt and fill out Cond with the test + /// condition. + virtual Optional<bool> + createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, + SmallVectorImpl<MachineOperand> &Cond) = 0; + + /// Modify the loop such that the trip count is + /// OriginalTC + TripCountAdjust. + virtual void adjustTripCount(int TripCountAdjust) = 0; + + /// Called when the loop's preheader has been modified to NewPreheader. + virtual void setPreheader(MachineBasicBlock *NewPreheader) = 0; + + /// Called when the loop is being removed. Any instructions in the preheader + /// should be removed. + /// + /// Once this function is called, no other functions on this object are + /// valid; the loop has been removed. + virtual void disposed() = 0; + }; + + /// Analyze loop L, which must be a single-basic-block loop, and if the + /// conditions can be understood enough produce a PipelinerLoopInfo object. + virtual std::unique_ptr<PipelinerLoopInfo> + analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { + return nullptr; + } + /// Analyze the loop code, return true if it cannot be understoo. Upon /// success, this function returns false and returns information about the /// induction variable and compare instruction used at the end. @@ -730,6 +779,19 @@ public: return false; } + /// Return the increase in code size needed to predicate a contiguous run of + /// NumInsts instructions. + virtual unsigned extraSizeToPredicateInstructions(const MachineFunction &MF, + unsigned NumInsts) const { + return 0; + } + + /// Return an estimate for the code size reduction (in bytes) which will be + /// caused by removing the given branch instruction during if-conversion. + virtual unsigned predictBranchSizeForIfCvt(MachineInstr &MI) const { + return getInstSizeInBytes(MI); + } + /// Return true if it's profitable to unpredicate /// one side of a 'diamond', i.e. two sides of if-else predicated on mutually /// exclusive predicates. @@ -1558,8 +1620,7 @@ public: /// function. virtual bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, - const MachineInstr &MIb, - AliasAnalysis *AA = nullptr) const { + const MachineInstr &MIb) const { assert((MIa.mayLoad() || MIa.mayStore()) && "MIa must load from or modify a memory location"); assert((MIb.mayLoad() || MIb.mayStore()) && @@ -1636,6 +1697,28 @@ public: return false; } + /// During PHI eleimination lets target to make necessary checks and + /// insert the copy to the PHI destination register in a target specific + /// manner. + virtual MachineInstr *createPHIDestinationCopy( + MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, + const DebugLoc &DL, Register Src, Register Dst) const { + return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst) + .addReg(Src); + } + + /// During PHI eleimination lets target to make necessary checks and + /// insert the copy to the PHI destination register in a target specific + /// manner. + virtual MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB, + MachineBasicBlock::iterator InsPt, + const DebugLoc &DL, Register Src, + Register SrcSubReg, + Register Dst) const { + return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst) + .addReg(Src, 0, SrcSubReg); + } + /// Returns a \p outliner::OutlinedFunction struct containing target-specific /// information for a set of outlining candidates. virtual outliner::OutlinedFunction getOutliningCandidateInfo( @@ -1691,6 +1774,11 @@ public: return false; } + /// Produce the expression describing the \p MI loading a value into + /// the parameter's forwarding register. + virtual Optional<ParamLoadedValue> + describeLoadedValue(const MachineInstr &MI) const; + private: unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode; unsigned CatchRetOpcode; diff --git a/include/llvm/CodeGen/TargetLowering.h b/include/llvm/CodeGen/TargetLowering.h index d5cca60bb1b2..a58fca7e73f5 100644 --- a/include/llvm/CodeGen/TargetLowering.h +++ b/include/llvm/CodeGen/TargetLowering.h @@ -28,7 +28,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/RuntimeLibcalls.h" @@ -48,6 +47,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Type.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -72,8 +72,10 @@ class Constant; class FastISel; class FunctionLoweringInfo; class GlobalValue; +class GISelKnownBits; class IntrinsicInst; struct KnownBits; +class LegacyDivergenceAnalysis; class LLVMContext; class MachineBasicBlock; class MachineFunction; @@ -122,8 +124,7 @@ public: TypeLegal, // The target natively supports this type. TypePromoteInteger, // Replace this integer with a larger one. TypeExpandInteger, // Split this integer into two of half the size. - TypeSoftenFloat, // Convert this float to a same size integer type, - // if an operation is not supported in target HW. + TypeSoftenFloat, // Convert this float to a same size integer type. TypeExpandFloat, // Split this float into two of half the size. TypeScalarizeVector, // Replace this one-element vector with its element. TypeSplitVector, // Split this vector into two of half the size. @@ -284,7 +285,7 @@ public: /// a constant pool load whose address depends on the select condition. The /// parameter may be used to differentiate a select with FP compare from /// integer compare. - virtual bool reduceSelectOfFPConstantLoads(bool IsFPSetCC) const { + virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const { return true; } @@ -539,6 +540,12 @@ public: return hasAndNotCompare(X); } + /// Return true if the target has a bit-test instruction: + /// (X & (1 << Y)) ==/!= 0 + /// This knowledge can be used to prevent breaking the pattern, + /// or creating it if it could be recognized. + virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; } + /// There are two ways to clear extreme bits (either low or high): /// Mask: x & (-1 << y) (the instcombine canonical form) /// Shifts: x >> y << y @@ -571,6 +578,38 @@ public: return false; } + /// Given the pattern + /// (X & (C l>>/<< Y)) ==/!= 0 + /// return true if it should be transformed into: + /// ((X <</l>> Y) & C) ==/!= 0 + /// WARNING: if 'X' is a constant, the fold may deadlock! + /// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat() + /// here because it can end up being not linked in. + virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd( + SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, + unsigned OldShiftOpcode, unsigned NewShiftOpcode, + SelectionDAG &DAG) const { + if (hasBitTest(X, Y)) { + // One interesting pattern that we'd want to form is 'bit test': + // ((1 << Y) & C) ==/!= 0 + // But we also need to be careful not to try to reverse that fold. + + // Is this '1 << Y' ? + if (OldShiftOpcode == ISD::SHL && CC->isOne()) + return false; // Keep the 'bit test' pattern. + + // Will it be '1 << Y' after the transform ? + if (XC && NewShiftOpcode == ISD::SHL && XC->isOne()) + return true; // Do form the 'bit test' pattern. + } + + // If 'X' is a constant, and we transform, then we will immediately + // try to undo the fold, thus causing endless combine loop. + // So by default, let's assume everyone prefers the fold + // iff 'X' is not a constant. + return !XC; + } + /// These two forms are equivalent: /// sub %y, (xor %x, -1) /// add (add %x, 1), %y @@ -798,9 +837,9 @@ public: PointerUnion<const Value *, const PseudoSourceValue *> ptrVal; int offset = 0; // offset off of ptrVal - unsigned size = 0; // the size of the memory location + uint64_t size = 0; // the size of the memory location // (taken from memVT if zero) - unsigned align = 1; // alignment + MaybeAlign align = Align::None(); // alignment MachineMemOperand::Flags flags = MachineMemOperand::MONone; IntrinsicInfo() = default; @@ -884,6 +923,7 @@ public: case ISD::SMULFIX: case ISD::SMULFIXSAT: case ISD::UMULFIX: + case ISD::UMULFIXSAT: Supported = isSupportedFixedPointOperation(Op, VT, Scale); break; } @@ -891,6 +931,8 @@ public: return Supported ? Action : Expand; } + // If Op is a strict floating-point operation, return the result + // of getOperationAction for the equivalent non-strict operation. LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const { unsigned EqOpc; switch (Op) { @@ -911,26 +953,25 @@ public: case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break; case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break; case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break; + case ISD::STRICT_LRINT: EqOpc = ISD::LRINT; break; + case ISD::STRICT_LLRINT: EqOpc = ISD::LLRINT; break; case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break; case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break; case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break; case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break; case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break; case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; + case ISD::STRICT_LROUND: EqOpc = ISD::LROUND; break; + case ISD::STRICT_LLROUND: EqOpc = ISD::LLROUND; break; case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; + case ISD::STRICT_FP_TO_SINT: EqOpc = ISD::FP_TO_SINT; break; + case ISD::STRICT_FP_TO_UINT: EqOpc = ISD::FP_TO_UINT; break; case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break; case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break; } - auto Action = getOperationAction(EqOpc, VT); - - // We don't currently handle Custom or Promote for strict FP pseudo-ops. - // For now, we just expand for those cases. - if (Action != Legal) - Action = Expand; - - return Action; + return getOperationAction(EqOpc, VT); } /// Return true if the specified operation is legal on this target or can be @@ -1206,7 +1247,7 @@ public: EltTy = PointerTy.getTypeForEVT(Ty->getContext()); } return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false), - VTy->getNumElements()); + VTy->getElementCount()); } return EVT::getEVT(Ty, AllowUnknown); @@ -1316,9 +1357,9 @@ public: /// Certain targets have context senstive alignment requirements, where one /// type has the alignment requirement of another type. - virtual unsigned getABIAlignmentForCallingConv(Type *ArgTy, - DataLayout DL) const { - return DL.getABITypeAlignment(ArgTy); + virtual Align getABIAlignmentForCallingConv(Type *ArgTy, + DataLayout DL) const { + return Align(DL.getABITypeAlignment(ArgTy)); } /// If true, then instruction selection should seek to shrink the FP constant @@ -1426,11 +1467,38 @@ public: return false; } + /// LLT handling variant. + virtual bool allowsMisalignedMemoryAccesses( + LLT, unsigned AddrSpace = 0, unsigned Align = 1, + MachineMemOperand::Flags Flags = MachineMemOperand::MONone, + bool * /*Fast*/ = nullptr) const { + return false; + } + + /// This function returns true if the memory access is aligned or if the + /// target allows this specific unaligned memory access. If the access is + /// allowed, the optional final parameter returns if the access is also fast + /// (as defined by the target). + bool allowsMemoryAccessForAlignment( + LLVMContext &Context, const DataLayout &DL, EVT VT, + unsigned AddrSpace = 0, unsigned Alignment = 1, + MachineMemOperand::Flags Flags = MachineMemOperand::MONone, + bool *Fast = nullptr) const; + + /// Return true if the memory access of this type is aligned or if the target + /// allows this specific unaligned access for the given MachineMemOperand. + /// If the access is allowed, the optional final parameter returns if the + /// access is also fast (as defined by the target). + bool allowsMemoryAccessForAlignment(LLVMContext &Context, + const DataLayout &DL, EVT VT, + const MachineMemOperand &MMO, + bool *Fast = nullptr) const; + /// Return true if the target supports a memory access of this type for the /// given address space and alignment. If the access is allowed, the optional /// final parameter returns if the access is also fast (as defined by the /// target). - bool + virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace = 0, unsigned Alignment = 1, MachineMemOperand::Flags Flags = MachineMemOperand::MONone, @@ -1463,6 +1531,16 @@ public: return MVT::Other; } + + /// LLT returning variant. + virtual LLT + getOptimalMemOpLLT(uint64_t /*Size*/, unsigned /*DstAlign*/, + unsigned /*SrcAlign*/, bool /*IsMemset*/, + bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/, + const AttributeList & /*FuncAttributes*/) const { + return LLT(); + } + /// Returns true if it's safe to use load / store of the specified type to /// expand memcpy / memset inline. /// @@ -1522,35 +1600,19 @@ public: report_fatal_error("Funclet EH is not implemented for this target"); } - /// Returns the target's jmp_buf size in bytes (if never set, the default is - /// 200) - unsigned getJumpBufSize() const { - return JumpBufSize; - } - - /// Returns the target's jmp_buf alignment in bytes (if never set, the default - /// is 0) - unsigned getJumpBufAlignment() const { - return JumpBufAlignment; - } - /// Return the minimum stack alignment of an argument. - unsigned getMinStackArgumentAlignment() const { + Align getMinStackArgumentAlignment() const { return MinStackArgumentAlignment; } /// Return the minimum function alignment. - unsigned getMinFunctionAlignment() const { - return MinFunctionAlignment; - } + Align getMinFunctionAlignment() const { return MinFunctionAlignment; } /// Return the preferred function alignment. - unsigned getPrefFunctionAlignment() const { - return PrefFunctionAlignment; - } + Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; } /// Return the preferred loop alignment. - virtual unsigned getPrefLoopAlignment(MachineLoop *ML = nullptr) const { + virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const { return PrefLoopAlignment; } @@ -1772,6 +1834,11 @@ public: return IsSigned; } + /// Returns true if arguments should be extended in lib calls. + virtual bool shouldExtendTypeInLibCall(EVT Type) const { + return true; + } + /// Returns how the given (atomic) load should be expanded by the /// IR-level AtomicExpand pass. virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const { @@ -1848,7 +1915,8 @@ public: /// This may be true if the target does not directly support the /// multiplication operation for the specified type or the sequence of simpler /// ops is faster than the multiply. - virtual bool decomposeMulByConstant(EVT VT, SDValue C) const { + virtual bool decomposeMulByConstant(LLVMContext &Context, + EVT VT, SDValue C) const { return false; } @@ -2056,40 +2124,25 @@ protected: TargetDAGCombineArray[NT >> 3] |= 1 << (NT&7); } - /// Set the target's required jmp_buf buffer size (in bytes); default is 200 - void setJumpBufSize(unsigned Size) { - JumpBufSize = Size; - } - - /// Set the target's required jmp_buf buffer alignment (in bytes); default is - /// 0 - void setJumpBufAlignment(unsigned Align) { - JumpBufAlignment = Align; - } - - /// Set the target's minimum function alignment (in log2(bytes)) - void setMinFunctionAlignment(unsigned Align) { - MinFunctionAlignment = Align; + /// Set the target's minimum function alignment. + void setMinFunctionAlignment(Align Alignment) { + MinFunctionAlignment = Alignment; } /// Set the target's preferred function alignment. This should be set if - /// there is a performance benefit to higher-than-minimum alignment (in - /// log2(bytes)) - void setPrefFunctionAlignment(unsigned Align) { - PrefFunctionAlignment = Align; + /// there is a performance benefit to higher-than-minimum alignment + void setPrefFunctionAlignment(Align Alignment) { + PrefFunctionAlignment = Alignment; } - /// Set the target's preferred loop alignment. Default alignment is zero, it - /// means the target does not care about loop alignment. The alignment is - /// specified in log2(bytes). The target may also override - /// getPrefLoopAlignment to provide per-loop values. - void setPrefLoopAlignment(unsigned Align) { - PrefLoopAlignment = Align; - } + /// Set the target's preferred loop alignment. Default alignment is one, it + /// means the target does not care about loop alignment. The target may also + /// override getPrefLoopAlignment to provide per-loop values. + void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; } - /// Set the minimum stack alignment of an argument (in log2(bytes)). - void setMinStackArgumentAlignment(unsigned Align) { - MinStackArgumentAlignment = Align; + /// Set the minimum stack alignment of an argument. + void setMinStackArgumentAlignment(Align Alignment) { + MinStackArgumentAlignment = Alignment; } /// Set the maximum atomic operation size supported by the @@ -2555,6 +2608,12 @@ public: // same blocks of its users. virtual bool shouldConsiderGEPOffsetSplit() const { return false; } + // Return the shift amount threshold for profitable transforms into shifts. + // Transforms creating shifts above the returned value will be avoided. + virtual unsigned getShiftAmountThreshold(EVT VT) const { + return VT.getScalarSizeInBits(); + } + //===--------------------------------------------------------------------===// // Runtime Library hooks // @@ -2650,25 +2709,19 @@ private: /// register usage. Sched::Preference SchedPreferenceInfo; - /// The size, in bytes, of the target's jmp_buf buffers - unsigned JumpBufSize; - - /// The alignment, in bytes, of the target's jmp_buf buffers - unsigned JumpBufAlignment; - /// The minimum alignment that any argument on the stack needs to have. - unsigned MinStackArgumentAlignment; + Align MinStackArgumentAlignment; /// The minimum function alignment (used when optimizing for size, and to /// prevent explicitly provided alignment from leading to incorrect code). - unsigned MinFunctionAlignment; + Align MinFunctionAlignment; /// The preferred function alignment (used when alignment unspecified and /// optimizing for speed). - unsigned PrefFunctionAlignment; + Align PrefFunctionAlignment; - /// The preferred loop alignment. - unsigned PrefLoopAlignment; + /// The preferred loop alignment (in log2 bot in bytes). + Align PrefLoopAlignment; /// Size in bits of the maximum atomics size the backend supports. /// Accesses larger than this will be expanded by AtomicExpandPass. @@ -2744,7 +2797,6 @@ private: /// up the MVT::LAST_VALUETYPE value to the next multiple of 8. uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::LAST_VALUETYPE + 7) / 8]; -protected: ValueTypeActionImpl ValueTypeActions; private: @@ -2790,7 +2842,7 @@ protected: /// expected to be merged. unsigned GatherAllAliasesMaxDepth; - /// Specify maximum number of store instructions per memset call. + /// \brief Specify maximum number of store instructions per memset call. /// /// When lowering \@llvm.memset this field specifies the maximum number of /// store operations that may be substituted for the call to memset. Targets @@ -2801,12 +2853,10 @@ protected: /// with 16-bit alignment would result in four 2-byte stores and one 1-byte /// store. This only applies to setting a constant array of a constant size. unsigned MaxStoresPerMemset; - - /// Maximum number of stores operations that may be substituted for the call - /// to memset, used for functions with OptSize attribute. + /// Likewise for functions with the OptSize attribute. unsigned MaxStoresPerMemsetOptSize; - /// Specify maximum bytes of store instructions per memcpy call. + /// \brief Specify maximum number of store instructions per memcpy call. /// /// When lowering \@llvm.memcpy this field specifies the maximum number of /// store operations that may be substituted for a call to memcpy. Targets @@ -2818,8 +2868,8 @@ protected: /// and one 1-byte store. This only applies to copying a constant array of /// constant size. unsigned MaxStoresPerMemcpy; - - + /// Likewise for functions with the OptSize attribute. + unsigned MaxStoresPerMemcpyOptSize; /// \brief Specify max number of store instructions to glue in inlined memcpy. /// /// When memcpy is inlined based on MaxStoresPerMemcpy, specify maximum number @@ -2827,13 +2877,22 @@ protected: // vectorization later on. unsigned MaxGluedStoresPerMemcpy = 0; - /// Maximum number of store operations that may be substituted for a call to - /// memcpy, used for functions with OptSize attribute. - unsigned MaxStoresPerMemcpyOptSize; + /// \brief Specify maximum number of load instructions per memcmp call. + /// + /// When lowering \@llvm.memcmp this field specifies the maximum number of + /// pairs of load operations that may be substituted for a call to memcmp. + /// Targets must set this value based on the cost threshold for that target. + /// Targets should assume that the memcmp will be done using as many of the + /// largest load operations first, followed by smaller ones, if necessary, per + /// alignment restrictions. For example, loading 7 bytes on a 32-bit machine + /// with 32-bit alignment would result in one 4-byte load, a one 2-byte load + /// and one 1-byte load. This only applies to copying a constant array of + /// constant size. unsigned MaxLoadsPerMemcmp; + /// Likewise for functions with the OptSize attribute. unsigned MaxLoadsPerMemcmpOptSize; - /// Specify maximum bytes of store instructions per memmove call. + /// \brief Specify maximum number of store instructions per memmove call. /// /// When lowering \@llvm.memmove this field specifies the maximum number of /// store instructions that may be substituted for a call to memmove. Targets @@ -2844,9 +2903,7 @@ protected: /// with 8-bit alignment would result in nine 1-byte stores. This only /// applies to copying a constant array of constant size. unsigned MaxStoresPerMemmove; - - /// Maximum number of store instructions that may be substituted for a call to - /// memmove, used for functions with OptSize attribute. + /// Likewise for functions with the OptSize attribute. unsigned MaxStoresPerMemmoveOptSize; /// Tells the code generator that select is more expensive than a branch if @@ -2885,6 +2942,7 @@ protected: class TargetLowering : public TargetLoweringBase { public: struct DAGCombinerInfo; + struct MakeLibCallOptions; TargetLowering(const TargetLowering &) = delete; TargetLowering &operator=(const TargetLowering &) = delete; @@ -2925,6 +2983,14 @@ public: return false; } + /// Returns true if the specified base+offset is a legal indexed addressing + /// mode for this target. \p MI is the load or store instruction that is being + /// considered for transformation. + virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset, + bool IsPre, MachineRegisterInfo &MRI) const { + return false; + } + /// Return the entry encoding for a jump table in the current function. The /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum. virtual unsigned getJumpTableEncoding() const; @@ -2955,14 +3021,15 @@ public: void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, - const SDLoc &DL) const; + const SDLoc &DL, const SDValue OldLHS, + const SDValue OldRHS) const; /// Returns a pair of (return value, chain). /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC. - std::pair<SDValue, SDValue> makeLibCall( - SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops, - bool isSigned, const SDLoc &dl, bool doesNotReturn = false, - bool isReturnValueUsed = true, bool isPostTypeLegalization = false) const; + std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, + EVT RetVT, ArrayRef<SDValue> Ops, + MakeLibCallOptions CallOptions, + const SDLoc &dl) const; /// Check whether parameters to a call that are passed in callee saved /// registers are the same as from the calling function. This needs to be @@ -3065,6 +3132,14 @@ public: bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, DAGCombinerInfo &DCI) const; + /// More limited version of SimplifyDemandedBits that can be used to "look + /// through" ops that don't contribute to the DemandedBits/DemandedElts - + /// bitwise ops etc. + SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts, + SelectionDAG &DAG, + unsigned Depth) const; + /// Look at Vector Op. At this point, we know that only the DemandedElts /// elements of the result of Op are ever used downstream. If we can use /// this information to simplify Op, create a new simplified DAG node and @@ -3099,6 +3174,15 @@ public: const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth = 0) const; + /// Determine which of the bits specified in Mask are known to be either zero + /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts + /// argument allows us to only collect the known bits that are shared by the + /// requested vector elements. This is for GISel. + virtual void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis, + Register R, KnownBits &Known, + const APInt &DemandedElts, + const MachineRegisterInfo &MRI, + unsigned Depth = 0) const; /// Determine which of the bits of FrameIndex \p FIOp are known to be 0. /// Default implementation computes low bits based on alignment @@ -3139,6 +3223,21 @@ public: TargetLoweringOpt &TLO, unsigned Depth = 0) const; + /// More limited version of SimplifyDemandedBits that can be used to "look + /// through" ops that don't contribute to the DemandedBits/DemandedElts - + /// bitwise ops etc. + virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode( + SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + SelectionDAG &DAG, unsigned Depth) const; + + /// Tries to build a legal vector shuffle using the provided parameters + /// or equivalent variations. The Mask argument maybe be modified as the + /// function tries different variations. + /// Returns an empty SDValue if the operation fails. + SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, + SDValue N1, MutableArrayRef<int> Mask, + SelectionDAG &DAG) const; + /// This method returns the constant pool value that will be loaded by LD. /// NOTE: You must check for implicit extensions of the constant by LD. virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const; @@ -3174,6 +3273,8 @@ public: SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true); SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true); + bool recursivelyDeleteUnusedNodes(SDNode *N); + void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO); }; @@ -3297,6 +3398,18 @@ public: llvm_unreachable("Not Implemented"); } + /// Return 1 if we can compute the negated form of the specified expression + /// for the same cost as the expression itself, or 2 if we can compute the + /// negated form more cheaply than the expression itself. Else return 0. + virtual char isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth = 0) const; + + /// If isNegatibleForFree returns true, return the newly negated expression. + virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth = 0) const; + //===--------------------------------------------------------------------===// // Lowering methods - These methods must be implemented by targets so that // the SelectionDAGBuilder code knows how to lower these. @@ -3468,6 +3581,51 @@ public: } }; + /// This structure is used to pass arguments to makeLibCall function. + struct MakeLibCallOptions { + // By passing type list before soften to makeLibCall, the target hook + // shouldExtendTypeInLibCall can get the original type before soften. + ArrayRef<EVT> OpsVTBeforeSoften; + EVT RetVTBeforeSoften; + bool IsSExt : 1; + bool DoesNotReturn : 1; + bool IsReturnValueUsed : 1; + bool IsPostTypeLegalization : 1; + bool IsSoften : 1; + + MakeLibCallOptions() + : IsSExt(false), DoesNotReturn(false), IsReturnValueUsed(true), + IsPostTypeLegalization(false), IsSoften(false) {} + + MakeLibCallOptions &setSExt(bool Value = true) { + IsSExt = Value; + return *this; + } + + MakeLibCallOptions &setNoReturn(bool Value = true) { + DoesNotReturn = Value; + return *this; + } + + MakeLibCallOptions &setDiscardResult(bool Value = true) { + IsReturnValueUsed = !Value; + return *this; + } + + MakeLibCallOptions &setIsPostTypeLegalization(bool Value = true) { + IsPostTypeLegalization = Value; + return *this; + } + + MakeLibCallOptions &setTypeListBeforeSoften(ArrayRef<EVT> OpsVT, EVT RetVT, + bool Value = true) { + OpsVTBeforeSoften = OpsVT; + RetVTBeforeSoften = RetVT; + IsSoften = Value; + return *this; + } + }; + /// This function lowers an abstract call to a function into an actual call. /// This returns a pair of operands. The first element is the return value /// for the function (if RetTy is not VoidTy). The second element is the @@ -3537,8 +3695,8 @@ public: /// Return the register ID of the name passed in. Used by named register /// global variables extension. There is no target-independent behaviour /// so the default action is to bail. - virtual unsigned getRegisterByName(const char* RegName, EVT VT, - SelectionDAG &DAG) const { + virtual Register getRegisterByName(const char* RegName, EVT VT, + const MachineFunction &MF) const { report_fatal_error("Named registers not implemented for this target"); } @@ -3597,6 +3755,25 @@ public: return MachineMemOperand::MONone; } + /// Should SelectionDAG lower an atomic store of the given kind as a normal + /// StoreSDNode (as opposed to an AtomicSDNode)? NOTE: The intention is to + /// eventually migrate all targets to the using StoreSDNodes, but porting is + /// being done target at a time. + virtual bool lowerAtomicStoreAsStoreSDNode(const StoreInst &SI) const { + assert(SI.isAtomic() && "violated precondition"); + return false; + } + + /// Should SelectionDAG lower an atomic load of the given kind as a normal + /// LoadSDNode (as opposed to an AtomicSDNode)? NOTE: The intention is to + /// eventually migrate all targets to the using LoadSDNodes, but porting is + /// being done target at a time. + virtual bool lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const { + assert(LI.isAtomic() && "violated precondition"); + return false; + } + + /// This callback is invoked by the type legalizer to legalize nodes with an /// illegal operand type but legal result types. It replaces the /// LowerOperation callback in the type Legalizer. The reason we can not do @@ -3665,6 +3842,7 @@ public: C_Register, // Constraint represents specific register(s). C_RegisterClass, // Constraint represents any of register(s) in class. C_Memory, // Memory constraint. + C_Immediate, // Requires an immediate. C_Other, // Something else. C_Unknown // Unsupported constraint. }; @@ -3905,7 +4083,7 @@ public: /// \param N Node to expand /// \param Result output after conversion /// \returns True, if the expansion was successful, false otherwise - bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const; /// Expand UINT(i64) to double(f64) conversion /// \param N Node to expand @@ -3986,8 +4164,8 @@ public: /// method accepts integers as its arguments. SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const; - /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts - /// integers as its arguments. + /// Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT]. This + /// method accepts integers as its arguments. SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const; /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion @@ -4070,6 +4248,11 @@ private: DAGCombinerInfo &DCI, const SDLoc &DL) const; + // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0 + SDValue optimizeSetCCByHoistingAndByConstFromLogicalShift( + EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, + DAGCombinerInfo &DCI, const SDLoc &DL) const; + SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL, @@ -4077,6 +4260,14 @@ private: SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const; + + SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, + SDValue CompTargetNode, ISD::CondCode Cond, + DAGCombinerInfo &DCI, const SDLoc &DL, + SmallVectorImpl<SDNode *> &Created) const; + SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, + ISD::CondCode Cond, DAGCombinerInfo &DCI, + const SDLoc &DL) const; }; /// Given an LLVM IR type and return type attributes, compute the return value diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index a1fb81cb009d..59f5ddbd9dac 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H #define LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H +#include "llvm/BinaryFormat/XCOFF.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCExpr.h" #include "llvm/Target/TargetLoweringObjectFile.h" @@ -35,7 +36,7 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile { protected: MCSymbolRefExpr::VariantKind PLTRelativeVariantKind = MCSymbolRefExpr::VK_None; - const TargetMachine *TM; + const TargetMachine *TM = nullptr; public: TargetLoweringObjectFileELF() = default; @@ -126,7 +127,8 @@ public: MachineModuleInfo *MMI) const override; /// Get MachO PC relative GOT entry relocation - const MCExpr *getIndirectSymViaGOTPCRel(const MCSymbol *Sym, + const MCExpr *getIndirectSymViaGOTPCRel(const GlobalValue *GV, + const MCSymbol *Sym, const MCValue &MV, int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const override; @@ -206,6 +208,34 @@ public: const TargetMachine &TM) const override; }; +class TargetLoweringObjectFileXCOFF : public TargetLoweringObjectFile { +public: + TargetLoweringObjectFileXCOFF() = default; + ~TargetLoweringObjectFileXCOFF() override = default; + + void Initialize(MCContext &Ctx, const TargetMachine &TM) override; + + bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference, + const Function &F) const override; + + MCSection *getExplicitSectionGlobal(const GlobalObject *GO, SectionKind Kind, + const TargetMachine &TM) const override; + + MCSection *getStaticCtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + MCSection *getStaticDtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + + const MCExpr *lowerRelativeReference(const GlobalValue *LHS, + const GlobalValue *RHS, + const TargetMachine &TM) const override; + + MCSection *SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind, + const TargetMachine &TM) const override; + + static XCOFF::StorageClass getStorageClassForGlobal(const GlobalObject *GO); +}; + } // end namespace llvm #endif // LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H diff --git a/include/llvm/CodeGen/TargetPassConfig.h b/include/llvm/CodeGen/TargetPassConfig.h index 0bd82aafac37..d48fc664c1c3 100644 --- a/include/llvm/CodeGen/TargetPassConfig.h +++ b/include/llvm/CodeGen/TargetPassConfig.h @@ -280,7 +280,7 @@ public: /// /// This can also be used to plug a new MachineSchedStrategy into an instance /// of the standard ScheduleDAGMI: - /// return new ScheduleDAGMI(C, make_unique<MyStrategy>(C), /*RemoveKillFlags=*/false) + /// return new ScheduleDAGMI(C, std::make_unique<MyStrategy>(C), /*RemoveKillFlags=*/false) /// /// Return NULL to select the default (generic) machine scheduler. virtual ScheduleDAGInstrs * diff --git a/include/llvm/CodeGen/TargetRegisterInfo.h b/include/llvm/CodeGen/TargetRegisterInfo.h index ddbd677b3eaa..c42ca3ad6eb9 100644 --- a/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/include/llvm/CodeGen/TargetRegisterInfo.h @@ -87,11 +87,20 @@ public: /// Return true if the specified register is included in this register class. /// This does not include virtual registers. bool contains(unsigned Reg) const { + /// FIXME: Historically this function has returned false when given vregs + /// but it should probably only receive physical registers + if (!Register::isPhysicalRegister(Reg)) + return false; return MC->contains(Reg); } /// Return true if both registers are in this class. bool contains(unsigned Reg1, unsigned Reg2) const { + /// FIXME: Historically this function has returned false when given a vregs + /// but it should probably only receive physical registers + if (!Register::isPhysicalRegister(Reg1) || + !Register::isPhysicalRegister(Reg2)) + return false; return MC->contains(Reg1, Reg2); } @@ -258,57 +267,6 @@ public: // Further sentinels can be allocated from the small negative integers. // DenseMapInfo<unsigned> uses -1u and -2u. - /// isStackSlot - Sometimes it is useful the be able to store a non-negative - /// frame index in a variable that normally holds a register. isStackSlot() - /// returns true if Reg is in the range used for stack slots. - /// - /// Note that isVirtualRegister() and isPhysicalRegister() cannot handle stack - /// slots, so if a variable may contains a stack slot, always check - /// isStackSlot() first. - /// - static bool isStackSlot(unsigned Reg) { - return int(Reg) >= (1 << 30); - } - - /// Compute the frame index from a register value representing a stack slot. - static int stackSlot2Index(unsigned Reg) { - assert(isStackSlot(Reg) && "Not a stack slot"); - return int(Reg - (1u << 30)); - } - - /// Convert a non-negative frame index to a stack slot register value. - static unsigned index2StackSlot(int FI) { - assert(FI >= 0 && "Cannot hold a negative frame index."); - return FI + (1u << 30); - } - - /// Return true if the specified register number is in - /// the physical register namespace. - static bool isPhysicalRegister(unsigned Reg) { - assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); - return int(Reg) > 0; - } - - /// Return true if the specified register number is in - /// the virtual register namespace. - static bool isVirtualRegister(unsigned Reg) { - assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); - return int(Reg) < 0; - } - - /// Convert a virtual register number to a 0-based index. - /// The first virtual register in a function will get the index 0. - static unsigned virtReg2Index(unsigned Reg) { - assert(isVirtualRegister(Reg) && "Not a virtual register"); - return Reg & ~(1u << 31); - } - - /// Convert a 0-based index to a virtual register number. - /// This is the inverse operation of VirtReg2IndexFunctor below. - static unsigned index2VirtReg(unsigned Index) { - return Index | (1u << 31); - } - /// Return the size in bits of a register from class RC. unsigned getRegSizeInBits(const TargetRegisterClass &RC) const { return getRegClassInfo(RC).RegSize; @@ -419,9 +377,9 @@ public: /// Returns true if the two registers are equal or alias each other. /// The registers may be virtual registers. - bool regsOverlap(unsigned regA, unsigned regB) const { + bool regsOverlap(Register regA, Register regB) const { if (regA == regB) return true; - if (isVirtualRegister(regA) || isVirtualRegister(regB)) + if (regA.isVirtual() || regB.isVirtual()) return false; // Regunits are numerically ordered. Find a common unit. @@ -489,6 +447,14 @@ public: llvm_unreachable("target does not provide no preserved mask"); } + /// Return a list of all of the registers which are clobbered "inside" a call + /// to the given function. For example, these might be needed for PLT + /// sequences of long-branch veneers. + virtual ArrayRef<MCPhysReg> + getIntraCallClobberedRegs(const MachineFunction *MF) const { + return {}; + } + /// Return true if all bits that are set in mask \p mask0 are also set in /// \p mask1. bool regmaskSubsetEqual(const uint32_t *mask0, const uint32_t *mask1) const; @@ -535,6 +501,11 @@ public: return false; } + /// This is a wrapper around getCallPreservedMask(). + /// Return true if the register is preserved after the call. + virtual bool isCalleeSavedPhysReg(unsigned PhysReg, + const MachineFunction &MF) const; + /// Prior to adding the live-out mask to a stackmap or patchpoint /// instruction, provide the target the opportunity to adjust it (mainly to /// remove pseudo-registers that should be ignored). @@ -709,13 +680,9 @@ public: /// Find the largest common subclass of A and B. /// Return NULL if there is no common subclass. - /// The common subclass should contain - /// simple value type SVT if it is not the Any type. const TargetRegisterClass * getCommonSubClass(const TargetRegisterClass *A, - const TargetRegisterClass *B, - const MVT::SimpleValueType SVT = - MVT::SimpleValueType::Any) const; + const TargetRegisterClass *B) const; /// Returns a TargetRegisterClass used for pointer values. /// If a target supports multiple different pointer register classes, @@ -1005,6 +972,13 @@ public: const MachineRegisterInfo &MRI) const { return nullptr; } + + /// Returns the physical register number of sub-register "Index" + /// for physical register RegNo. Return zero if the sub-register does not + /// exist. + inline Register getSubReg(MCRegister Reg, unsigned Idx) const { + return static_cast<const MCRegisterInfo *>(this)->getSubReg(Reg, Idx); + } }; //===----------------------------------------------------------------------===// @@ -1156,7 +1130,7 @@ public: struct VirtReg2IndexFunctor { using argument_type = unsigned; unsigned operator()(unsigned Reg) const { - return TargetRegisterInfo::virtReg2Index(Reg); + return Register::virtReg2Index(Reg); } }; @@ -1170,7 +1144,7 @@ struct VirtReg2IndexFunctor { /// %physreg17 - a physical register when no TRI instance given. /// /// Usage: OS << printReg(Reg, TRI, SubRegIdx) << '\n'; -Printable printReg(unsigned Reg, const TargetRegisterInfo *TRI = nullptr, +Printable printReg(Register Reg, const TargetRegisterInfo *TRI = nullptr, unsigned SubIdx = 0, const MachineRegisterInfo *MRI = nullptr); diff --git a/include/llvm/CodeGen/TargetSubtargetInfo.h b/include/llvm/CodeGen/TargetSubtargetInfo.h index 037fc3ed3243..56018eca8c27 100644 --- a/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -106,12 +106,10 @@ public: // us do things like a dedicated avx512 selector). However, we might want // to also specialize selectors by MachineFunction, which would let us be // aware of optsize/optnone and such. - virtual const InstructionSelector *getInstructionSelector() const { + virtual InstructionSelector *getInstructionSelector() const { return nullptr; } - virtual unsigned getHwMode() const { return 0; } - /// Target can subclass this hook to select a different DAG scheduler. virtual RegisterScheduler::FunctionPassCtor getDAGScheduler(CodeGenOpt::Level) const { @@ -274,6 +272,12 @@ public: /// scheduling, DAGCombine, etc.). virtual bool useAA() const; + /// \brief Sink addresses into blocks using GEP instructions rather than + /// pointer casts and arithmetic. + virtual bool addrSinkUsingGEPs() const { + return useAA(); + } + /// Enable the use of the early if conversion pass. virtual bool enableEarlyIfConversion() const { return false; } diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index c540c94f79d9..cd4c4ca64081 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -81,7 +81,7 @@ namespace llvm { /// Returns the EVT that represents a vector EC.Min elements in length, /// where each element is of type VT. - static EVT getVectorVT(LLVMContext &Context, EVT VT, MVT::ElementCount EC) { + static EVT getVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) { MVT M = MVT::getVectorVT(VT.V, EC); if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE) return M; @@ -277,7 +277,7 @@ namespace llvm { } // Given a (possibly scalable) vector type, return the ElementCount - MVT::ElementCount getVectorElementCount() const { + ElementCount getVectorElementCount() const { assert((isVector()) && "Invalid vector type!"); if (isSimple()) return V.getVectorElementCount(); diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td index 5818ac183fcc..16df565bc8b8 100644 --- a/include/llvm/CodeGen/ValueTypes.td +++ b/include/llvm/CodeGen/ValueTypes.td @@ -40,127 +40,132 @@ def v16i1 : ValueType<16, 18>; // 16 x i1 vector value def v32i1 : ValueType<32 , 19>; // 32 x i1 vector value def v64i1 : ValueType<64 , 20>; // 64 x i1 vector value def v128i1 : ValueType<128, 21>; // 128 x i1 vector value -def v512i1 : ValueType<512, 22>; // 512 x i1 vector value -def v1024i1: ValueType<1024,23>; //1024 x i1 vector value - -def v1i8 : ValueType<8, 24>; // 1 x i8 vector value -def v2i8 : ValueType<16 , 25>; // 2 x i8 vector value -def v4i8 : ValueType<32 , 26>; // 4 x i8 vector value -def v8i8 : ValueType<64 , 27>; // 8 x i8 vector value -def v16i8 : ValueType<128, 28>; // 16 x i8 vector value -def v32i8 : ValueType<256, 29>; // 32 x i8 vector value -def v64i8 : ValueType<512, 30>; // 64 x i8 vector value -def v128i8 : ValueType<1024,31>; //128 x i8 vector value -def v256i8 : ValueType<2048,32>; //256 x i8 vector value - -def v1i16 : ValueType<16 , 33>; // 1 x i16 vector value -def v2i16 : ValueType<32 , 34>; // 2 x i16 vector value -def v4i16 : ValueType<64 , 35>; // 4 x i16 vector value -def v8i16 : ValueType<128, 36>; // 8 x i16 vector value -def v16i16 : ValueType<256, 37>; // 16 x i16 vector value -def v32i16 : ValueType<512, 38>; // 32 x i16 vector value -def v64i16 : ValueType<1024,39>; // 64 x i16 vector value -def v128i16: ValueType<2048,40>; //128 x i16 vector value - -def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value -def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value -def v3i32 : ValueType<96 , 43>; // 3 x i32 vector value -def v4i32 : ValueType<128, 44>; // 4 x i32 vector value -def v5i32 : ValueType<160, 45>; // 5 x i32 vector value -def v8i32 : ValueType<256, 46>; // 8 x i32 vector value -def v16i32 : ValueType<512, 47>; // 16 x i32 vector value -def v32i32 : ValueType<1024,48>; // 32 x i32 vector value -def v64i32 : ValueType<2048,49>; // 64 x i32 vector value -def v128i32 : ValueType<4096,50>; // 128 x i32 vector value -def v256i32 : ValueType<8182,51>; // 256 x i32 vector value -def v512i32 : ValueType<16384,52>; // 512 x i32 vector value -def v1024i32 : ValueType<32768,53>; // 1024 x i32 vector value -def v2048i32 : ValueType<65536,54>; // 2048 x i32 vector value - -def v1i64 : ValueType<64 , 55>; // 1 x i64 vector value -def v2i64 : ValueType<128, 56>; // 2 x i64 vector value -def v4i64 : ValueType<256, 57>; // 4 x i64 vector value -def v8i64 : ValueType<512, 58>; // 8 x i64 vector value -def v16i64 : ValueType<1024,59>; // 16 x i64 vector value -def v32i64 : ValueType<2048,60>; // 32 x i64 vector value - -def v1i128 : ValueType<128, 61>; // 1 x i128 vector value - -def nxv1i1 : ValueType<1, 62>; // n x 1 x i1 vector value -def nxv2i1 : ValueType<2, 63>; // n x 2 x i1 vector value -def nxv4i1 : ValueType<4, 64>; // n x 4 x i1 vector value -def nxv8i1 : ValueType<8, 65>; // n x 8 x i1 vector value -def nxv16i1 : ValueType<16, 66>; // n x 16 x i1 vector value -def nxv32i1 : ValueType<32, 67>; // n x 32 x i1 vector value - -def nxv1i8 : ValueType<8, 68>; // n x 1 x i8 vector value -def nxv2i8 : ValueType<16, 69>; // n x 2 x i8 vector value -def nxv4i8 : ValueType<32, 70>; // n x 4 x i8 vector value -def nxv8i8 : ValueType<64, 71>; // n x 8 x i8 vector value -def nxv16i8 : ValueType<128, 72>; // n x 16 x i8 vector value -def nxv32i8 : ValueType<256, 73>; // n x 32 x i8 vector value - -def nxv1i16 : ValueType<16, 74>; // n x 1 x i16 vector value -def nxv2i16 : ValueType<32, 75>; // n x 2 x i16 vector value -def nxv4i16 : ValueType<64, 76>; // n x 4 x i16 vector value -def nxv8i16 : ValueType<128, 77>; // n x 8 x i16 vector value -def nxv16i16: ValueType<256, 78>; // n x 16 x i16 vector value -def nxv32i16: ValueType<512, 79>; // n x 32 x i16 vector value - -def nxv1i32 : ValueType<32, 80>; // n x 1 x i32 vector value -def nxv2i32 : ValueType<64, 81>; // n x 2 x i32 vector value -def nxv4i32 : ValueType<128, 82>; // n x 4 x i32 vector value -def nxv8i32 : ValueType<256, 83>; // n x 8 x i32 vector value -def nxv16i32: ValueType<512, 84>; // n x 16 x i32 vector value -def nxv32i32: ValueType<1024,85>; // n x 32 x i32 vector value - -def nxv1i64 : ValueType<64, 86>; // n x 1 x i64 vector value -def nxv2i64 : ValueType<128, 87>; // n x 2 x i64 vector value -def nxv4i64 : ValueType<256, 88>; // n x 4 x i64 vector value -def nxv8i64 : ValueType<512, 89>; // n x 8 x i64 vector value -def nxv16i64: ValueType<1024,90>; // n x 16 x i64 vector value -def nxv32i64: ValueType<2048,91>; // n x 32 x i64 vector value - -def v2f16 : ValueType<32 , 92>; // 2 x f16 vector value -def v4f16 : ValueType<64 , 93>; // 4 x f16 vector value -def v8f16 : ValueType<128, 94>; // 8 x f16 vector value -def v1f32 : ValueType<32 , 95>; // 1 x f32 vector value -def v2f32 : ValueType<64 , 96>; // 2 x f32 vector value -def v3f32 : ValueType<96 , 97>; // 3 x f32 vector value -def v4f32 : ValueType<128, 98>; // 4 x f32 vector value -def v5f32 : ValueType<160, 99>; // 5 x f32 vector value -def v8f32 : ValueType<256, 100>; // 8 x f32 vector value -def v16f32 : ValueType<512, 101>; // 16 x f32 vector value -def v32f32 : ValueType<1024, 102>; // 32 x f32 vector value -def v64f32 : ValueType<2048, 103>; // 64 x f32 vector value -def v128f32 : ValueType<4096, 104>; // 128 x f32 vector value -def v256f32 : ValueType<8182, 105>; // 256 x f32 vector value -def v512f32 : ValueType<16384, 106>; // 512 x f32 vector value -def v1024f32 : ValueType<32768, 107>; // 1024 x f32 vector value -def v2048f32 : ValueType<65536, 108>; // 2048 x f32 vector value -def v1f64 : ValueType<64, 109>; // 1 x f64 vector value -def v2f64 : ValueType<128, 110>; // 2 x f64 vector value -def v4f64 : ValueType<256, 111>; // 4 x f64 vector value -def v8f64 : ValueType<512, 112>; // 8 x f64 vector value - -def nxv2f16 : ValueType<32 , 113>; // n x 2 x f16 vector value -def nxv4f16 : ValueType<64 , 114>; // n x 4 x f16 vector value -def nxv8f16 : ValueType<128, 115>; // n x 8 x f16 vector value -def nxv1f32 : ValueType<32 , 116>; // n x 1 x f32 vector value -def nxv2f32 : ValueType<64 , 117>; // n x 2 x f32 vector value -def nxv4f32 : ValueType<128, 118>; // n x 4 x f32 vector value -def nxv8f32 : ValueType<256, 119>; // n x 8 x f32 vector value -def nxv16f32 : ValueType<512, 120>; // n x 16 x f32 vector value -def nxv1f64 : ValueType<64, 121>; // n x 1 x f64 vector value -def nxv2f64 : ValueType<128, 122>; // n x 2 x f64 vector value -def nxv4f64 : ValueType<256, 123>; // n x 4 x f64 vector value -def nxv8f64 : ValueType<512, 124>; // n x 8 x f64 vector value - -def x86mmx : ValueType<64 , 125>; // X86 MMX value -def FlagVT : ValueType<0 , 126>; // Pre-RA sched glue -def isVoid : ValueType<0 , 127>; // Produces no value -def untyped: ValueType<8 , 128>; // Produces an untyped value -def exnref: ValueType<0, 129>; // WebAssembly's exnref type +def v256i1 : ValueType<256, 22>; // 256 x i1 vector value +def v512i1 : ValueType<512, 23>; // 512 x i1 vector value +def v1024i1: ValueType<1024,24>; //1024 x i1 vector value + +def v1i8 : ValueType<8, 25>; // 1 x i8 vector value +def v2i8 : ValueType<16 , 26>; // 2 x i8 vector value +def v4i8 : ValueType<32 , 27>; // 4 x i8 vector value +def v8i8 : ValueType<64 , 28>; // 8 x i8 vector value +def v16i8 : ValueType<128, 29>; // 16 x i8 vector value +def v32i8 : ValueType<256, 30>; // 32 x i8 vector value +def v64i8 : ValueType<512, 31>; // 64 x i8 vector value +def v128i8 : ValueType<1024,32>; //128 x i8 vector value +def v256i8 : ValueType<2048,33>; //256 x i8 vector value + +def v1i16 : ValueType<16 , 34>; // 1 x i16 vector value +def v2i16 : ValueType<32 , 35>; // 2 x i16 vector value +def v3i16 : ValueType<48 , 36>; // 3 x i16 vector value +def v4i16 : ValueType<64 , 37>; // 4 x i16 vector value +def v8i16 : ValueType<128, 38>; // 8 x i16 vector value +def v16i16 : ValueType<256, 39>; // 16 x i16 vector value +def v32i16 : ValueType<512, 40>; // 32 x i16 vector value +def v64i16 : ValueType<1024,41>; // 64 x i16 vector value +def v128i16: ValueType<2048,42>; //128 x i16 vector value + +def v1i32 : ValueType<32 , 43>; // 1 x i32 vector value +def v2i32 : ValueType<64 , 44>; // 2 x i32 vector value +def v3i32 : ValueType<96 , 45>; // 3 x i32 vector value +def v4i32 : ValueType<128, 46>; // 4 x i32 vector value +def v5i32 : ValueType<160, 47>; // 5 x i32 vector value +def v8i32 : ValueType<256, 48>; // 8 x i32 vector value +def v16i32 : ValueType<512, 49>; // 16 x i32 vector value +def v32i32 : ValueType<1024,50>; // 32 x i32 vector value +def v64i32 : ValueType<2048,51>; // 64 x i32 vector value +def v128i32 : ValueType<4096,52>; // 128 x i32 vector value +def v256i32 : ValueType<8182,53>; // 256 x i32 vector value +def v512i32 : ValueType<16384,54>; // 512 x i32 vector value +def v1024i32 : ValueType<32768,55>; // 1024 x i32 vector value +def v2048i32 : ValueType<65536,56>; // 2048 x i32 vector value + +def v1i64 : ValueType<64 , 57>; // 1 x i64 vector value +def v2i64 : ValueType<128, 58>; // 2 x i64 vector value +def v4i64 : ValueType<256, 59>; // 4 x i64 vector value +def v8i64 : ValueType<512, 60>; // 8 x i64 vector value +def v16i64 : ValueType<1024,61>; // 16 x i64 vector value +def v32i64 : ValueType<2048,62>; // 32 x i64 vector value + +def v1i128 : ValueType<128, 63>; // 1 x i128 vector value + +def v2f16 : ValueType<32 , 64>; // 2 x f16 vector value +def v3f16 : ValueType<48 , 65>; // 3 x f16 vector value +def v4f16 : ValueType<64 , 66>; // 4 x f16 vector value +def v8f16 : ValueType<128, 67>; // 8 x f16 vector value +def v16f16 : ValueType<256, 68>; // 8 x f16 vector value +def v32f16 : ValueType<512, 69>; // 8 x f16 vector value +def v1f32 : ValueType<32 , 70>; // 1 x f32 vector value +def v2f32 : ValueType<64 , 71>; // 2 x f32 vector value +def v3f32 : ValueType<96 , 72>; // 3 x f32 vector value +def v4f32 : ValueType<128, 73>; // 4 x f32 vector value +def v5f32 : ValueType<160, 74>; // 5 x f32 vector value +def v8f32 : ValueType<256, 75>; // 8 x f32 vector value +def v16f32 : ValueType<512, 76>; // 16 x f32 vector value +def v32f32 : ValueType<1024, 77>; // 32 x f32 vector value +def v64f32 : ValueType<2048, 78>; // 64 x f32 vector value +def v128f32 : ValueType<4096, 79>; // 128 x f32 vector value +def v256f32 : ValueType<8182, 80>; // 256 x f32 vector value +def v512f32 : ValueType<16384, 81>; // 512 x f32 vector value +def v1024f32 : ValueType<32768, 82>; // 1024 x f32 vector value +def v2048f32 : ValueType<65536, 83>; // 2048 x f32 vector value +def v1f64 : ValueType<64, 84>; // 1 x f64 vector value +def v2f64 : ValueType<128, 85>; // 2 x f64 vector value +def v4f64 : ValueType<256, 86>; // 4 x f64 vector value +def v8f64 : ValueType<512, 87>; // 8 x f64 vector value + +def nxv1i1 : ValueType<1, 88>; // n x 1 x i1 vector value +def nxv2i1 : ValueType<2, 89>; // n x 2 x i1 vector value +def nxv4i1 : ValueType<4, 90>; // n x 4 x i1 vector value +def nxv8i1 : ValueType<8, 91>; // n x 8 x i1 vector value +def nxv16i1 : ValueType<16, 92>; // n x 16 x i1 vector value +def nxv32i1 : ValueType<32, 93>; // n x 32 x i1 vector value + +def nxv1i8 : ValueType<8, 94>; // n x 1 x i8 vector value +def nxv2i8 : ValueType<16, 95>; // n x 2 x i8 vector value +def nxv4i8 : ValueType<32, 96>; // n x 4 x i8 vector value +def nxv8i8 : ValueType<64, 97>; // n x 8 x i8 vector value +def nxv16i8 : ValueType<128, 98>; // n x 16 x i8 vector value +def nxv32i8 : ValueType<256, 99>; // n x 32 x i8 vector value + +def nxv1i16 : ValueType<16, 100>; // n x 1 x i16 vector value +def nxv2i16 : ValueType<32, 101>; // n x 2 x i16 vector value +def nxv4i16 : ValueType<64, 102>; // n x 4 x i16 vector value +def nxv8i16 : ValueType<128, 103>; // n x 8 x i16 vector value +def nxv16i16: ValueType<256, 104>; // n x 16 x i16 vector value +def nxv32i16: ValueType<512, 105>; // n x 32 x i16 vector value + +def nxv1i32 : ValueType<32, 106>; // n x 1 x i32 vector value +def nxv2i32 : ValueType<64, 107>; // n x 2 x i32 vector value +def nxv4i32 : ValueType<128, 108>; // n x 4 x i32 vector value +def nxv8i32 : ValueType<256, 109>; // n x 8 x i32 vector value +def nxv16i32: ValueType<512, 110>; // n x 16 x i32 vector value +def nxv32i32: ValueType<1024,111>; // n x 32 x i32 vector value + +def nxv1i64 : ValueType<64, 112>; // n x 1 x i64 vector value +def nxv2i64 : ValueType<128, 113>; // n x 2 x i64 vector value +def nxv4i64 : ValueType<256, 114>; // n x 4 x i64 vector value +def nxv8i64 : ValueType<512, 115>; // n x 8 x i64 vector value +def nxv16i64: ValueType<1024,116>; // n x 16 x i64 vector value +def nxv32i64: ValueType<2048,117>; // n x 32 x i64 vector value + +def nxv2f16 : ValueType<32 , 118>; // n x 2 x f16 vector value +def nxv4f16 : ValueType<64 , 119>; // n x 4 x f16 vector value +def nxv8f16 : ValueType<128, 120>; // n x 8 x f16 vector value +def nxv1f32 : ValueType<32 , 121>; // n x 1 x f32 vector value +def nxv2f32 : ValueType<64 , 122>; // n x 2 x f32 vector value +def nxv4f32 : ValueType<128, 123>; // n x 4 x f32 vector value +def nxv8f32 : ValueType<256, 124>; // n x 8 x f32 vector value +def nxv16f32 : ValueType<512, 125>; // n x 16 x f32 vector value +def nxv1f64 : ValueType<64, 126>; // n x 1 x f64 vector value +def nxv2f64 : ValueType<128, 127>; // n x 2 x f64 vector value +def nxv4f64 : ValueType<256, 128>; // n x 4 x f64 vector value +def nxv8f64 : ValueType<512, 129>; // n x 8 x f64 vector value + +def x86mmx : ValueType<64 , 130>; // X86 MMX value +def FlagVT : ValueType<0 , 131>; // Pre-RA sched glue +def isVoid : ValueType<0 , 132>; // Produces no value +def untyped: ValueType<8 , 133>; // Produces an untyped value +def exnref: ValueType<0, 134>; // WebAssembly's exnref type def token : ValueType<0 , 248>; // TokenTy def MetadataVT: ValueType<0, 249>; // Metadata diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h index 70eb048f05eb..db25ed5c5116 100644 --- a/include/llvm/CodeGen/VirtRegMap.h +++ b/include/llvm/CodeGen/VirtRegMap.h @@ -49,7 +49,7 @@ class TargetInstrInfo; /// it; even spilled virtual registers (the register mapped to a /// spilled register is the temporary used to load it from the /// stack). - IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysMap; + IndexedMap<Register, VirtReg2IndexFunctor> Virt2PhysMap; /// Virt2StackSlotMap - This is virtual register to stack slot /// mapping. Each spilled virtual register has an entry in it @@ -93,7 +93,7 @@ class TargetInstrInfo; /// returns true if the specified virtual register is /// mapped to a physical register - bool hasPhys(unsigned virtReg) const { + bool hasPhys(Register virtReg) const { return getPhys(virtReg) != NO_PHYS_REG; } @@ -101,20 +101,20 @@ class TargetInstrInfo; /// virtual register Register getPhys(Register virtReg) const { assert(virtReg.isVirtual()); - return Virt2PhysMap[virtReg]; + return Virt2PhysMap[virtReg.id()]; } /// creates a mapping for the specified virtual register to /// the specified physical register - void assignVirt2Phys(unsigned virtReg, MCPhysReg physReg); + void assignVirt2Phys(Register virtReg, MCPhysReg physReg); /// clears the specified virtual register's, physical /// register mapping - void clearVirt(unsigned virtReg) { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); - assert(Virt2PhysMap[virtReg] != NO_PHYS_REG && + void clearVirt(Register virtReg) { + assert(virtReg.isVirtual()); + assert(Virt2PhysMap[virtReg.id()] != NO_PHYS_REG && "attempt to clear a not assigned virtual register"); - Virt2PhysMap[virtReg] = NO_PHYS_REG; + Virt2PhysMap[virtReg.id()] = NO_PHYS_REG; } /// clears all virtual to physical register mappings @@ -124,21 +124,21 @@ class TargetInstrInfo; } /// returns true if VirtReg is assigned to its preferred physreg. - bool hasPreferredPhys(unsigned VirtReg); + bool hasPreferredPhys(Register VirtReg); /// returns true if VirtReg has a known preferred register. /// This returns false if VirtReg has a preference that is a virtual /// register that hasn't been assigned yet. - bool hasKnownPreference(unsigned VirtReg); + bool hasKnownPreference(Register VirtReg); /// records virtReg is a split live interval from SReg. - void setIsSplitFromReg(unsigned virtReg, unsigned SReg) { - Virt2SplitMap[virtReg] = SReg; + void setIsSplitFromReg(Register virtReg, unsigned SReg) { + Virt2SplitMap[virtReg.id()] = SReg; } /// returns the live interval virtReg is split from. - unsigned getPreSplitReg(unsigned virtReg) const { - return Virt2SplitMap[virtReg]; + unsigned getPreSplitReg(Register virtReg) const { + return Virt2SplitMap[virtReg.id()]; } /// getOriginal - Return the original virtual register that VirtReg descends @@ -152,28 +152,29 @@ class TargetInstrInfo; /// returns true if the specified virtual register is not /// mapped to a stack slot or rematerialized. - bool isAssignedReg(unsigned virtReg) const { + bool isAssignedReg(Register virtReg) const { if (getStackSlot(virtReg) == NO_STACK_SLOT) return true; // Split register can be assigned a physical register as well as a // stack slot or remat id. - return (Virt2SplitMap[virtReg] && Virt2PhysMap[virtReg] != NO_PHYS_REG); + return (Virt2SplitMap[virtReg.id()] && + Virt2PhysMap[virtReg.id()] != NO_PHYS_REG); } /// returns the stack slot mapped to the specified virtual /// register - int getStackSlot(unsigned virtReg) const { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); - return Virt2StackSlotMap[virtReg]; + int getStackSlot(Register virtReg) const { + assert(virtReg.isVirtual()); + return Virt2StackSlotMap[virtReg.id()]; } /// create a mapping for the specifed virtual register to /// the next available stack slot - int assignVirt2StackSlot(unsigned virtReg); + int assignVirt2StackSlot(Register virtReg); /// create a mapping for the specified virtual register to /// the specified stack slot - void assignVirt2StackSlot(unsigned virtReg, int SS); + void assignVirt2StackSlot(Register virtReg, int SS); void print(raw_ostream &OS, const Module* M = nullptr) const override; void dump() const; diff --git a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h index 7d20bb0a7bde..7538cb2c2548 100644 --- a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h +++ b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h @@ -11,7 +11,6 @@ #include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" #include "llvm/Support/Error.h" namespace llvm { @@ -31,9 +30,6 @@ enum VisitorDataSource { Error visitTypeRecord(CVType &Record, TypeIndex Index, TypeVisitorCallbacks &Callbacks, VisitorDataSource Source = VDS_BytesPresent); -Error visitTypeRecord(CVType &Record, TypeIndex Index, - TypeVisitorCallbackPipeline &Callbacks, - VisitorDataSource Source = VDS_BytesPresent); Error visitTypeRecord(CVType &Record, TypeVisitorCallbacks &Callbacks, VisitorDataSource Source = VDS_BytesPresent); diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h index 00fb0cf4cc90..60829a51dc25 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h +++ b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h @@ -33,6 +33,9 @@ public: virtual void EmitIntValue(uint64_t Value, unsigned Size) = 0; virtual void EmitBinaryData(StringRef Data) = 0; virtual void AddComment(const Twine &T) = 0; + virtual void AddRawComment(const Twine &T) = 0; + virtual bool isVerboseAsm() = 0; + virtual std::string getTypeName(TypeIndex TI) = 0; virtual ~CodeViewRecordStreamer() = default; }; @@ -206,6 +209,11 @@ public: return 0; } + void emitRawComment(const Twine &T) { + if (isStreaming() && Streamer->isVerboseAsm()) + Streamer->AddRawComment(T); + } + private: void emitEncodedSignedInteger(const int64_t &Value, const Twine &Comment = ""); @@ -225,9 +233,10 @@ private: } void emitComment(const Twine &Comment) { - if (isStreaming()) { + if (isStreaming() && Streamer->isVerboseAsm()) { Twine TComment(Comment); - Streamer->AddComment(TComment); + if (!TComment.isTriviallyEmpty()) + Streamer->AddComment(TComment); } } diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def index 9767e49c44f5..ed5c143818e6 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def +++ b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def @@ -366,8 +366,134 @@ CV_REGISTER(AMD64_K7, 765) #endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_X86) +#if defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM) + +// ARM registers + +CV_REGISTER(ARM_NOREG, 0) + +// General purpose 32-bit integer regisers + +CV_REGISTER(ARM_R0, 10) +CV_REGISTER(ARM_R1, 11) +CV_REGISTER(ARM_R2, 12) +CV_REGISTER(ARM_R3, 13) +CV_REGISTER(ARM_R4, 14) +CV_REGISTER(ARM_R5, 15) +CV_REGISTER(ARM_R6, 16) +CV_REGISTER(ARM_R7, 17) +CV_REGISTER(ARM_R8, 18) +CV_REGISTER(ARM_R9, 19) +CV_REGISTER(ARM_R10, 20) +CV_REGISTER(ARM_R11, 21) +CV_REGISTER(ARM_R12, 22) +CV_REGISTER(ARM_SP, 23) +CV_REGISTER(ARM_LR, 24) +CV_REGISTER(ARM_PC, 25) + +// Status register + +CV_REGISTER(ARM_CPSR, 25) + +// ARM VFPv1 registers + +CV_REGISTER(ARM_FPSCR, 40) +CV_REGISTER(ARM_FPEXC, 41) + +// ARM VFPv3/NEON registers + +CV_REGISTER(ARM_FS32, 200) +CV_REGISTER(ARM_FS33, 201) +CV_REGISTER(ARM_FS34, 202) +CV_REGISTER(ARM_FS35, 203) +CV_REGISTER(ARM_FS36, 204) +CV_REGISTER(ARM_FS37, 205) +CV_REGISTER(ARM_FS38, 206) +CV_REGISTER(ARM_FS39, 207) +CV_REGISTER(ARM_FS40, 208) +CV_REGISTER(ARM_FS41, 209) +CV_REGISTER(ARM_FS42, 210) +CV_REGISTER(ARM_FS43, 211) +CV_REGISTER(ARM_FS44, 212) +CV_REGISTER(ARM_FS45, 213) +CV_REGISTER(ARM_FS46, 214) +CV_REGISTER(ARM_FS47, 215) +CV_REGISTER(ARM_FS48, 216) +CV_REGISTER(ARM_FS49, 217) +CV_REGISTER(ARM_FS50, 218) +CV_REGISTER(ARM_FS51, 219) +CV_REGISTER(ARM_FS52, 220) +CV_REGISTER(ARM_FS53, 221) +CV_REGISTER(ARM_FS54, 222) +CV_REGISTER(ARM_FS55, 223) +CV_REGISTER(ARM_FS56, 224) +CV_REGISTER(ARM_FS57, 225) +CV_REGISTER(ARM_FS58, 226) +CV_REGISTER(ARM_FS59, 227) +CV_REGISTER(ARM_FS60, 228) +CV_REGISTER(ARM_FS61, 229) +CV_REGISTER(ARM_FS62, 230) +CV_REGISTER(ARM_FS63, 231) + +CV_REGISTER(ARM_ND0, 300) +CV_REGISTER(ARM_ND1, 301) +CV_REGISTER(ARM_ND2, 302) +CV_REGISTER(ARM_ND3, 303) +CV_REGISTER(ARM_ND4, 304) +CV_REGISTER(ARM_ND5, 305) +CV_REGISTER(ARM_ND6, 306) +CV_REGISTER(ARM_ND7, 307) +CV_REGISTER(ARM_ND8, 308) +CV_REGISTER(ARM_ND9, 309) +CV_REGISTER(ARM_ND10, 310) +CV_REGISTER(ARM_ND11, 311) +CV_REGISTER(ARM_ND12, 312) +CV_REGISTER(ARM_ND13, 313) +CV_REGISTER(ARM_ND14, 314) +CV_REGISTER(ARM_ND15, 315) +CV_REGISTER(ARM_ND16, 316) +CV_REGISTER(ARM_ND17, 317) +CV_REGISTER(ARM_ND18, 318) +CV_REGISTER(ARM_ND19, 319) +CV_REGISTER(ARM_ND20, 320) +CV_REGISTER(ARM_ND21, 321) +CV_REGISTER(ARM_ND22, 322) +CV_REGISTER(ARM_ND23, 323) +CV_REGISTER(ARM_ND24, 324) +CV_REGISTER(ARM_ND25, 325) +CV_REGISTER(ARM_ND26, 326) +CV_REGISTER(ARM_ND27, 327) +CV_REGISTER(ARM_ND28, 328) +CV_REGISTER(ARM_ND29, 329) +CV_REGISTER(ARM_ND30, 330) +CV_REGISTER(ARM_ND31, 331) + +CV_REGISTER(ARM_NQ0, 400) +CV_REGISTER(ARM_NQ1, 401) +CV_REGISTER(ARM_NQ2, 402) +CV_REGISTER(ARM_NQ3, 403) +CV_REGISTER(ARM_NQ4, 404) +CV_REGISTER(ARM_NQ5, 405) +CV_REGISTER(ARM_NQ6, 406) +CV_REGISTER(ARM_NQ7, 407) +CV_REGISTER(ARM_NQ8, 408) +CV_REGISTER(ARM_NQ9, 409) +CV_REGISTER(ARM_NQ10, 410) +CV_REGISTER(ARM_NQ11, 411) +CV_REGISTER(ARM_NQ12, 412) +CV_REGISTER(ARM_NQ13, 413) +CV_REGISTER(ARM_NQ14, 414) +CV_REGISTER(ARM_NQ15, 415) + +#endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM) + #if defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64) +// arm64intr.h from MSVC defines ARM64_FPSR, which conflicts with +// these declarations. +#pragma push_macro("ARM64_FPSR") +#undef ARM64_FPSR + // ARM64 registers CV_REGISTER(ARM64_NOREG, 0) @@ -556,4 +682,6 @@ CV_REGISTER(ARM64_Q31, 211) CV_REGISTER(ARM64_FPSR, 220) +#pragma pop_macro("ARM64_FPSR") + #endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64) diff --git a/include/llvm/DebugInfo/CodeView/EnumTables.h b/include/llvm/DebugInfo/CodeView/EnumTables.h index ed126ed9e2ff..270cd4b8330c 100644 --- a/include/llvm/DebugInfo/CodeView/EnumTables.h +++ b/include/llvm/DebugInfo/CodeView/EnumTables.h @@ -37,6 +37,17 @@ ArrayRef<EnumEntry<uint8_t>> getThunkOrdinalNames(); ArrayRef<EnumEntry<uint16_t>> getTrampolineNames(); ArrayRef<EnumEntry<COFF::SectionCharacteristics>> getImageSectionCharacteristicNames(); +ArrayRef<EnumEntry<uint16_t>> getClassOptionNames(); +ArrayRef<EnumEntry<uint8_t>> getMemberAccessNames(); +ArrayRef<EnumEntry<uint16_t>> getMethodOptionNames(); +ArrayRef<EnumEntry<uint16_t>> getMemberKindNames(); +ArrayRef<EnumEntry<uint8_t>> getPtrKindNames(); +ArrayRef<EnumEntry<uint8_t>> getPtrModeNames(); +ArrayRef<EnumEntry<uint16_t>> getPtrMemberRepNames(); +ArrayRef<EnumEntry<uint16_t>> getTypeModifierNames(); +ArrayRef<EnumEntry<uint8_t>> getCallingConventions(); +ArrayRef<EnumEntry<uint8_t>> getFunctionOptionEnum(); +ArrayRef<EnumEntry<uint16_t>> getLabelTypeEnum(); } // end namespace codeview } // end namespace llvm diff --git a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h index 62761cb87c81..108abb291498 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h @@ -62,7 +62,7 @@ public: Error visitSymbolBegin(CVSymbol &Record) override { assert(!Mapping && "Already in a symbol mapping!"); - Mapping = llvm::make_unique<MappingInfo>(Record.content(), Container); + Mapping = std::make_unique<MappingInfo>(Record.content(), Container); return Mapping->Mapping.visitSymbolBegin(Record); } Error visitSymbolEnd(CVSymbol &Record) override { diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/include/llvm/DebugInfo/CodeView/SymbolRecord.h index 5e9a7432b9b6..1aafa3ca9f1d 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -73,17 +73,17 @@ public: Thunk32Sym(SymbolRecordKind Kind, uint32_t RecordOffset) : SymbolRecord(Kind), RecordOffset(RecordOffset) {} - uint32_t Parent; - uint32_t End; - uint32_t Next; - uint32_t Offset; - uint16_t Segment; - uint16_t Length; + uint32_t Parent = 0; + uint32_t End = 0; + uint32_t Next = 0; + uint32_t Offset = 0; + uint16_t Segment = 0; + uint16_t Length = 0; ThunkOrdinal Thunk; StringRef Name; ArrayRef<uint8_t> VariantData; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_TRAMPOLINE @@ -94,13 +94,13 @@ public: : SymbolRecord(Kind), RecordOffset(RecordOffset) {} TrampolineType Type; - uint16_t Size; - uint32_t ThunkOffset; - uint32_t TargetOffset; - uint16_t ThunkSection; - uint16_t TargetSection; + uint16_t Size = 0; + uint32_t ThunkOffset = 0; + uint32_t TargetOffset = 0; + uint16_t ThunkSection = 0; + uint16_t TargetSection = 0; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_SECTION @@ -110,14 +110,14 @@ public: SectionSym(SymbolRecordKind Kind, uint32_t RecordOffset) : SymbolRecord(Kind), RecordOffset(RecordOffset) {} - uint16_t SectionNumber; - uint8_t Alignment; - uint32_t Rva; - uint32_t Length; - uint32_t Characteristics; + uint16_t SectionNumber = 0; + uint8_t Alignment = 0; + uint32_t Rva = 0; + uint32_t Length = 0; + uint32_t Characteristics = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_COFFGROUP @@ -127,13 +127,13 @@ public: CoffGroupSym(SymbolRecordKind Kind, uint32_t RecordOffset) : SymbolRecord(Kind), RecordOffset(RecordOffset) {} - uint32_t Size; - uint32_t Characteristics; - uint32_t Offset; - uint16_t Segment; + uint32_t Size = 0; + uint32_t Characteristics = 0; + uint32_t Offset = 0; + uint16_t Segment = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; class ScopeEndSym : public SymbolRecord { @@ -142,7 +142,7 @@ public: ScopeEndSym(SymbolRecordKind Kind, uint32_t RecordOffset) : SymbolRecord(Kind), RecordOffset(RecordOffset) {} - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; class CallerSym : public SymbolRecord { @@ -153,7 +153,7 @@ public: std::vector<TypeIndex> Indices; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; struct DecodedAnnotation { @@ -333,7 +333,7 @@ private: class InlineSiteSym : public SymbolRecord { public: explicit InlineSiteSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - InlineSiteSym(uint32_t RecordOffset) + explicit InlineSiteSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::InlineSiteSym), RecordOffset(RecordOffset) {} @@ -342,12 +342,12 @@ public: BinaryAnnotationIterator()); } - uint32_t Parent; - uint32_t End; + uint32_t Parent = 0; + uint32_t End = 0; TypeIndex Inlinee; std::vector<uint8_t> AnnotationData; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_PUB32 @@ -371,7 +371,7 @@ public: class RegisterSym : public SymbolRecord { public: explicit RegisterSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - RegisterSym(uint32_t RecordOffset) + explicit RegisterSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::RegisterSym), RecordOffset(RecordOffset) {} @@ -379,7 +379,7 @@ public: RegisterId Register; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_PROCREF, S_LPROCREF @@ -390,13 +390,13 @@ public: : SymbolRecord(SymbolRecordKind::ProcRefSym), RecordOffset(RecordOffset) { } - uint32_t SumName; - uint32_t SymOffset; - uint16_t Module; + uint32_t SumName = 0; + uint32_t SymOffset = 0; + uint16_t Module = 0; StringRef Name; uint16_t modi() const { return Module - 1; } - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_LOCAL @@ -410,7 +410,7 @@ public: LocalSymFlags Flags; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; struct LocalVariableAddrRange { @@ -440,11 +440,11 @@ public: return RecordOffset + RelocationOffset; } - uint32_t Program; + uint32_t Program = 0; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_DEFRANGE_SUBFIELD @@ -453,7 +453,7 @@ class DefRangeSubfieldSym : public SymbolRecord { public: explicit DefRangeSubfieldSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - DefRangeSubfieldSym(uint32_t RecordOffset) + explicit DefRangeSubfieldSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DefRangeSubfieldSym), RecordOffset(RecordOffset) {} @@ -461,58 +461,62 @@ public: return RecordOffset + RelocationOffset; } - uint32_t Program; - uint16_t OffsetInParent; + uint32_t Program = 0; + uint16_t OffsetInParent = 0; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; +}; + +struct DefRangeRegisterHeader { + ulittle16_t Register; + ulittle16_t MayHaveNoName; }; // S_DEFRANGE_REGISTER class DefRangeRegisterSym : public SymbolRecord { public: - struct Header { - ulittle16_t Register; - ulittle16_t MayHaveNoName; - }; - explicit DefRangeRegisterSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - DefRangeRegisterSym(uint32_t RecordOffset) + explicit DefRangeRegisterSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DefRangeRegisterSym), RecordOffset(RecordOffset) {} - uint32_t getRelocationOffset() const { return RecordOffset + sizeof(Header); } + uint32_t getRelocationOffset() const { return RecordOffset + sizeof(DefRangeRegisterHeader); } - Header Hdr; + DefRangeRegisterHeader Hdr; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; +}; + +struct DefRangeSubfieldRegisterHeader { + ulittle16_t Register; + ulittle16_t MayHaveNoName; + ulittle32_t OffsetInParent; }; // S_DEFRANGE_SUBFIELD_REGISTER class DefRangeSubfieldRegisterSym : public SymbolRecord { public: - struct Header { - ulittle16_t Register; - ulittle16_t MayHaveNoName; - ulittle32_t OffsetInParent; - }; - explicit DefRangeSubfieldRegisterSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - DefRangeSubfieldRegisterSym(uint32_t RecordOffset) + explicit DefRangeSubfieldRegisterSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DefRangeSubfieldRegisterSym), RecordOffset(RecordOffset) {} - uint32_t getRelocationOffset() const { return RecordOffset + sizeof(Header); } + uint32_t getRelocationOffset() const { return RecordOffset + sizeof(DefRangeSubfieldRegisterHeader); } - Header Hdr; + DefRangeSubfieldRegisterHeader Hdr; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; +}; + +struct DefRangeFramePointerRelHeader { + little32_t Offset; }; // S_DEFRANGE_FRAMEPOINTER_REL @@ -522,7 +526,7 @@ class DefRangeFramePointerRelSym : public SymbolRecord { public: explicit DefRangeFramePointerRelSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - DefRangeFramePointerRelSym(uint32_t RecordOffset) + explicit DefRangeFramePointerRelSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DefRangeFramePointerRelSym), RecordOffset(RecordOffset) {} @@ -530,22 +534,22 @@ public: return RecordOffset + RelocationOffset; } - int32_t Offset; + DefRangeFramePointerRelHeader Hdr; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; +}; + +struct DefRangeRegisterRelHeader { + ulittle16_t Register; + ulittle16_t Flags; + little32_t BasePointerOffset; }; // S_DEFRANGE_REGISTER_REL class DefRangeRegisterRelSym : public SymbolRecord { public: - struct Header { - ulittle16_t Register; - ulittle16_t Flags; - little32_t BasePointerOffset; - }; - explicit DefRangeRegisterRelSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} explicit DefRangeRegisterRelSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DefRangeRegisterRelSym), @@ -563,13 +567,13 @@ public: bool hasSpilledUDTMember() const { return Hdr.Flags & IsSubfieldFlag; } uint16_t offsetInParent() const { return Hdr.Flags >> OffsetInParentShift; } - uint32_t getRelocationOffset() const { return RecordOffset + sizeof(Header); } + uint32_t getRelocationOffset() const { return RecordOffset + sizeof(DefRangeRegisterRelHeader); } - Header Hdr; + DefRangeRegisterRelHeader Hdr; LocalVariableAddrRange Range; std::vector<LocalVariableAddrGap> Gaps; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_DEFRANGE_FRAMEPOINTER_REL_FULL_SCOPE @@ -581,9 +585,9 @@ public: : SymbolRecord(SymbolRecordKind::DefRangeFramePointerRelFullScopeSym), RecordOffset(RecordOffset) {} - int32_t Offset; + int32_t Offset = 0; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_BLOCK32 @@ -599,14 +603,14 @@ public: return RecordOffset + RelocationOffset; } - uint32_t Parent; - uint32_t End; - uint32_t CodeSize; - uint32_t CodeOffset; - uint16_t Segment; + uint32_t Parent = 0; + uint32_t End = 0; + uint32_t CodeSize = 0; + uint32_t CodeOffset = 0; + uint16_t Segment = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_LABEL32 @@ -622,12 +626,12 @@ public: return RecordOffset + RelocationOffset; } - uint32_t CodeOffset; - uint16_t Segment; + uint32_t CodeOffset = 0; + uint16_t Segment = 0; ProcSymFlags Flags; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_OBJNAME @@ -635,82 +639,82 @@ class ObjNameSym : public SymbolRecord { public: explicit ObjNameSym() : SymbolRecord(SymbolRecordKind::ObjNameSym) {} explicit ObjNameSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - ObjNameSym(uint32_t RecordOffset) + explicit ObjNameSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::ObjNameSym), RecordOffset(RecordOffset) { } - uint32_t Signature; + uint32_t Signature = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_ENVBLOCK class EnvBlockSym : public SymbolRecord { public: explicit EnvBlockSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - EnvBlockSym(uint32_t RecordOffset) + explicit EnvBlockSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::EnvBlockSym), RecordOffset(RecordOffset) {} std::vector<StringRef> Fields; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_EXPORT class ExportSym : public SymbolRecord { public: explicit ExportSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - ExportSym(uint32_t RecordOffset) + explicit ExportSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::ExportSym), RecordOffset(RecordOffset) {} - uint16_t Ordinal; + uint16_t Ordinal = 0; ExportFlags Flags; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_FILESTATIC class FileStaticSym : public SymbolRecord { public: explicit FileStaticSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - FileStaticSym(uint32_t RecordOffset) + explicit FileStaticSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::FileStaticSym), RecordOffset(RecordOffset) {} TypeIndex Index; - uint32_t ModFilenameOffset; + uint32_t ModFilenameOffset = 0; LocalSymFlags Flags; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_COMPILE2 class Compile2Sym : public SymbolRecord { public: explicit Compile2Sym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - Compile2Sym(uint32_t RecordOffset) + explicit Compile2Sym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::Compile2Sym), RecordOffset(RecordOffset) {} CompileSym2Flags Flags; CPUType Machine; - uint16_t VersionFrontendMajor; - uint16_t VersionFrontendMinor; - uint16_t VersionFrontendBuild; - uint16_t VersionBackendMajor; - uint16_t VersionBackendMinor; - uint16_t VersionBackendBuild; + uint16_t VersionFrontendMajor = 0; + uint16_t VersionFrontendMinor = 0; + uint16_t VersionFrontendBuild = 0; + uint16_t VersionBackendMajor = 0; + uint16_t VersionBackendMinor = 0; + uint16_t VersionBackendBuild = 0; StringRef Version; std::vector<StringRef> ExtraStrings; uint8_t getLanguage() const { return static_cast<uint32_t>(Flags) & 0xFF; } uint32_t getFlags() const { return static_cast<uint32_t>(Flags) & ~0xFF; } - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_COMPILE3 @@ -718,20 +722,20 @@ class Compile3Sym : public SymbolRecord { public: Compile3Sym() : SymbolRecord(SymbolRecordKind::Compile3Sym) {} explicit Compile3Sym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - Compile3Sym(uint32_t RecordOffset) + explicit Compile3Sym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::Compile3Sym), RecordOffset(RecordOffset) {} CompileSym3Flags Flags; CPUType Machine; - uint16_t VersionFrontendMajor; - uint16_t VersionFrontendMinor; - uint16_t VersionFrontendBuild; - uint16_t VersionFrontendQFE; - uint16_t VersionBackendMajor; - uint16_t VersionBackendMinor; - uint16_t VersionBackendBuild; - uint16_t VersionBackendQFE; + uint16_t VersionFrontendMajor = 0; + uint16_t VersionFrontendMinor = 0; + uint16_t VersionFrontendBuild = 0; + uint16_t VersionFrontendQFE = 0; + uint16_t VersionBackendMajor = 0; + uint16_t VersionBackendMinor = 0; + uint16_t VersionBackendBuild = 0; + uint16_t VersionBackendQFE = 0; StringRef Version; void setLanguage(SourceLanguage Lang) { @@ -750,7 +754,7 @@ public: (getFlags() & (CompileSym3Flags::PGO | CompileSym3Flags::LTCG)); } - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_FRAMEPROC @@ -761,12 +765,12 @@ public: : SymbolRecord(SymbolRecordKind::FrameProcSym), RecordOffset(RecordOffset) {} - uint32_t TotalFrameBytes; - uint32_t PaddingFrameBytes; - uint32_t OffsetToPadding; - uint32_t BytesOfCalleeSavedRegisters; - uint32_t OffsetOfExceptionHandler; - uint16_t SectionIdOfExceptionHandler; + uint32_t TotalFrameBytes = 0; + uint32_t PaddingFrameBytes = 0; + uint32_t OffsetToPadding = 0; + uint32_t BytesOfCalleeSavedRegisters = 0; + uint32_t OffsetOfExceptionHandler = 0; + uint16_t SectionIdOfExceptionHandler = 0; FrameProcedureOptions Flags; /// Extract the register this frame uses to refer to local variables. @@ -781,7 +785,7 @@ public: EncodedFramePtrReg((uint32_t(Flags) >> 16U) & 0x3U), CPU); } - uint32_t RecordOffset; + uint32_t RecordOffset = 0; private: }; @@ -799,11 +803,11 @@ public: return RecordOffset + RelocationOffset; } - uint32_t CodeOffset; - uint16_t Segment; + uint32_t CodeOffset = 0; + uint16_t Segment = 0; TypeIndex Type; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_HEAPALLOCSITE @@ -820,12 +824,12 @@ public: return RecordOffset + RelocationOffset; } - uint32_t CodeOffset; - uint16_t Segment; - uint16_t CallInstructionSize; + uint32_t CodeOffset = 0; + uint16_t Segment = 0; + uint16_t CallInstructionSize = 0; TypeIndex Type; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_FRAMECOOKIE @@ -841,12 +845,12 @@ public: return RecordOffset + RelocationOffset; } - uint32_t CodeOffset; - uint16_t Register; + uint32_t CodeOffset = 0; + uint16_t Register = 0; FrameCookieKind CookieKind; - uint8_t Flags; + uint8_t Flags = 0; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_UDT, S_COBOLUDT @@ -859,20 +863,20 @@ public: TypeIndex Type; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_BUILDINFO class BuildInfoSym : public SymbolRecord { public: explicit BuildInfoSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - BuildInfoSym(uint32_t RecordOffset) + explicit BuildInfoSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::BuildInfoSym), RecordOffset(RecordOffset) {} TypeIndex BuildId; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_BPREL32 @@ -883,11 +887,11 @@ public: : SymbolRecord(SymbolRecordKind::BPRelativeSym), RecordOffset(RecordOffset) {} - int32_t Offset; + int32_t Offset = 0; TypeIndex Type; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_REGREL32 @@ -898,19 +902,19 @@ public: : SymbolRecord(SymbolRecordKind::RegRelativeSym), RecordOffset(RecordOffset) {} - uint32_t Offset; + uint32_t Offset = 0; TypeIndex Type; RegisterId Register; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_CONSTANT, S_MANCONSTANT class ConstantSym : public SymbolRecord { public: explicit ConstantSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - ConstantSym(uint32_t RecordOffset) + explicit ConstantSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::ConstantSym), RecordOffset(RecordOffset) {} @@ -918,7 +922,7 @@ public: APSInt Value; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_LDATA32, S_GDATA32, S_LMANDATA, S_GMANDATA @@ -927,7 +931,7 @@ class DataSym : public SymbolRecord { public: explicit DataSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} - DataSym(uint32_t RecordOffset) + explicit DataSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::DataSym), RecordOffset(RecordOffset) {} uint32_t getRelocationOffset() const { @@ -935,11 +939,11 @@ public: } TypeIndex Type; - uint32_t DataOffset; - uint16_t Segment; + uint32_t DataOffset = 0; + uint16_t Segment = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_LTHREAD32, S_GTHREAD32 @@ -957,11 +961,11 @@ public: } TypeIndex Type; - uint32_t DataOffset; - uint16_t Segment; + uint32_t DataOffset = 0; + uint16_t Segment = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_UNAMESPACE @@ -974,7 +978,7 @@ public: StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_ANNOTATION @@ -989,7 +993,7 @@ public: uint16_t Segment = 0; std::vector<StringRef> Strings; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; using CVSymbol = CVRecord<SymbolKind>; diff --git a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h index 081de32dd02c..2b17f5ccb13b 100644 --- a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h +++ b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h @@ -66,7 +66,7 @@ public: Error visitTypeBegin(CVType &Record) override { assert(!Mapping && "Already in a type mapping!"); - Mapping = llvm::make_unique<MappingInfo>(Record.content()); + Mapping = std::make_unique<MappingInfo>(Record.content()); return Mapping->Mapping.visitTypeBegin(Record); } diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h index 4c309c10ff0c..c6044d5138a8 100644 --- a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h +++ b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h @@ -10,6 +10,7 @@ #define LLVM_DEBUGINFO_CODEVIEW_TYPERECORDMAPPING_H #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" #include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h" #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" #include "llvm/Support/Error.h" diff --git a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h index 169715be2d52..fb0b579d6a06 100644 --- a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h +++ b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h @@ -82,11 +82,6 @@ public: Pipeline.push_back(&Callbacks); } - void addCallbackToPipelineFront(TypeVisitorCallbacks &Callbacks) { - auto CallBackItr = Pipeline.begin(); - Pipeline.insert(CallBackItr, &Callbacks); - } - #define TYPE_RECORD(EnumName, EnumVal, Name) \ Error visitKnownRecord(CVType &CVR, Name##Record &Record) override { \ return visitKnownRecordImpl(CVR, Record); \ diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h index d2a5318179eb..fbebfe634b63 100644 --- a/include/llvm/DebugInfo/DIContext.h +++ b/include/llvm/DebugInfo/DIContext.h @@ -28,6 +28,10 @@ namespace llvm { /// A format-neutral container for source line information. struct DILineInfo { + // DILineInfo contains "<invalid>" for function/filename it cannot fetch. + static constexpr const char *const BadString = "<invalid>"; + // Use "??" instead of "<invalid>" to make our output closer to addr2line. + static constexpr const char *const Addr2LineBadString = "??"; std::string FileName; std::string FunctionName; Optional<StringRef> Source; @@ -38,7 +42,7 @@ struct DILineInfo { // DWARF-specific. uint32_t Discriminator = 0; - DILineInfo() : FileName("<invalid>"), FunctionName("<invalid>") {} + DILineInfo() : FileName(BadString), FunctionName(BadString) {} bool operator==(const DILineInfo &RHS) const { return Line == RHS.Line && Column == RHS.Column && @@ -61,9 +65,9 @@ struct DILineInfo { void dump(raw_ostream &OS) { OS << "Line info: "; - if (FileName != "<invalid>") + if (FileName != BadString) OS << "file '" << FileName << "', "; - if (FunctionName != "<invalid>") + if (FunctionName != BadString) OS << "function '" << FunctionName << "', "; OS << "line " << Line << ", "; OS << "column " << Column << ", "; @@ -109,7 +113,7 @@ struct DIGlobal { uint64_t Start = 0; uint64_t Size = 0; - DIGlobal() : Name("<invalid>") {} + DIGlobal() : Name(DILineInfo::BadString) {} }; struct DILocal { @@ -289,7 +293,7 @@ public: LoadedObjectInfoHelper(Ts &&... Args) : Base(std::forward<Ts>(Args)...) {} std::unique_ptr<llvm::LoadedObjectInfo> clone() const override { - return llvm::make_unique<Derived>(static_cast<const Derived &>(*this)); + return std::make_unique<Derived>(static_cast<const Derived &>(*this)); } }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h index ccf2891c2e21..39ae53c4e7fe 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h @@ -130,11 +130,11 @@ public: /// \param Attr DWARF attribute to search for. /// \param U the DWARFUnit the contains the DIE. /// \returns Optional DWARF form value if the attribute was extracted. - Optional<DWARFFormValue> getAttributeValue(const uint32_t DIEOffset, + Optional<DWARFFormValue> getAttributeValue(const uint64_t DIEOffset, const dwarf::Attribute Attr, const DWARFUnit &U) const; - bool extract(DataExtractor Data, uint32_t* OffsetPtr); + bool extract(DataExtractor Data, uint64_t* OffsetPtr); void dump(raw_ostream &OS) const; // Return an optional byte size of all attribute data in this abbreviation diff --git a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h index 303375703d2e..c9042e593260 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h @@ -96,7 +96,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable { using AtomType = uint16_t; using Form = dwarf::Form; - uint32_t DIEOffsetBase; + uint64_t DIEOffsetBase; SmallVector<std::pair<AtomType, Form>, 3> Atoms; Optional<uint64_t> extractOffset(Optional<DWARFFormValue> Value) const; @@ -109,7 +109,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable { /// Returns true if we should continue scanning for entries or false if we've /// reached the last (sentinel) entry of encountered a parsing error. bool dumpName(ScopedPrinter &W, SmallVectorImpl<DWARFFormValue> &AtomForms, - uint32_t *DataOffset) const; + uint64_t *DataOffset) const; public: /// Apple-specific implementation of an Accelerator Entry. @@ -119,7 +119,7 @@ public: Entry(const HeaderData &Data); Entry() = default; - void extract(const AppleAcceleratorTable &AccelTable, uint32_t *Offset); + void extract(const AppleAcceleratorTable &AccelTable, uint64_t *Offset); public: Optional<uint64_t> getCUOffset() const override; @@ -143,7 +143,7 @@ public: class ValueIterator : public std::iterator<std::input_iterator_tag, Entry> { const AppleAcceleratorTable *AccelTable = nullptr; Entry Current; ///< The current entry. - unsigned DataOffset = 0; ///< Offset into the section. + uint64_t DataOffset = 0; ///< Offset into the section. unsigned Data = 0; ///< Current data entry. unsigned NumData = 0; ///< Number of data entries. @@ -151,7 +151,7 @@ public: void Next(); public: /// Construct a new iterator for the entries at \p DataOffset. - ValueIterator(const AppleAcceleratorTable &AccelTable, unsigned DataOffset); + ValueIterator(const AppleAcceleratorTable &AccelTable, uint64_t DataOffset); /// End marker. ValueIterator() = default; @@ -193,7 +193,7 @@ public: /// DieOffset is the offset into the .debug_info section for the DIE /// related to the input hash data offset. /// DieTag is the tag of the DIE - std::pair<uint32_t, dwarf::Tag> readAtoms(uint32_t &HashDataOffset); + std::pair<uint64_t, dwarf::Tag> readAtoms(uint64_t *HashDataOffset); void dump(raw_ostream &OS) const override; /// Look up all entries in the accelerator table matching \c Key. @@ -245,7 +245,7 @@ public: struct Header : public HeaderPOD { SmallString<8> AugmentationString; - Error extract(const DWARFDataExtractor &AS, uint32_t *Offset); + Error extract(const DWARFDataExtractor &AS, uint64_t *Offset); void dump(ScopedPrinter &W) const; }; @@ -354,12 +354,12 @@ public: DataExtractor StrData; uint32_t Index; - uint32_t StringOffset; - uint32_t EntryOffset; + uint64_t StringOffset; + uint64_t EntryOffset; public: NameTableEntry(const DataExtractor &StrData, uint32_t Index, - uint32_t StringOffset, uint32_t EntryOffset) + uint64_t StringOffset, uint64_t EntryOffset) : StrData(StrData), Index(Index), StringOffset(StringOffset), EntryOffset(EntryOffset) {} @@ -367,17 +367,17 @@ public: uint32_t getIndex() const { return Index; } /// Returns the offset of the name of the described entities. - uint32_t getStringOffset() const { return StringOffset; } + uint64_t getStringOffset() const { return StringOffset; } /// Return the string referenced by this name table entry or nullptr if the /// string offset is not valid. const char *getString() const { - uint32_t Off = StringOffset; + uint64_t Off = StringOffset; return StrData.getCStr(&Off); } /// Returns the offset of the first Entry in the list. - uint32_t getEntryOffset() const { return EntryOffset; } + uint64_t getEntryOffset() const { return EntryOffset; } }; /// Represents a single accelerator table within the DWARF v5 .debug_names @@ -389,40 +389,40 @@ public: // Base of the whole unit and of various important tables, as offsets from // the start of the section. - uint32_t Base; - uint32_t CUsBase; - uint32_t BucketsBase; - uint32_t HashesBase; - uint32_t StringOffsetsBase; - uint32_t EntryOffsetsBase; - uint32_t EntriesBase; + uint64_t Base; + uint64_t CUsBase; + uint64_t BucketsBase; + uint64_t HashesBase; + uint64_t StringOffsetsBase; + uint64_t EntryOffsetsBase; + uint64_t EntriesBase; void dumpCUs(ScopedPrinter &W) const; void dumpLocalTUs(ScopedPrinter &W) const; void dumpForeignTUs(ScopedPrinter &W) const; void dumpAbbreviations(ScopedPrinter &W) const; - bool dumpEntry(ScopedPrinter &W, uint32_t *Offset) const; + bool dumpEntry(ScopedPrinter &W, uint64_t *Offset) const; void dumpName(ScopedPrinter &W, const NameTableEntry &NTE, Optional<uint32_t> Hash) const; void dumpBucket(ScopedPrinter &W, uint32_t Bucket) const; - Expected<AttributeEncoding> extractAttributeEncoding(uint32_t *Offset); + Expected<AttributeEncoding> extractAttributeEncoding(uint64_t *Offset); Expected<std::vector<AttributeEncoding>> - extractAttributeEncodings(uint32_t *Offset); + extractAttributeEncodings(uint64_t *Offset); - Expected<Abbrev> extractAbbrev(uint32_t *Offset); + Expected<Abbrev> extractAbbrev(uint64_t *Offset); public: - NameIndex(const DWARFDebugNames &Section, uint32_t Base) + NameIndex(const DWARFDebugNames &Section, uint64_t Base) : Section(Section), Base(Base) {} /// Reads offset of compilation unit CU. CU is 0-based. - uint32_t getCUOffset(uint32_t CU) const; + uint64_t getCUOffset(uint32_t CU) const; uint32_t getCUCount() const { return Hdr.CompUnitCount; } /// Reads offset of local type unit TU, TU is 0-based. - uint32_t getLocalTUOffset(uint32_t TU) const; + uint64_t getLocalTUOffset(uint32_t TU) const; uint32_t getLocalTUCount() const { return Hdr.LocalTypeUnitCount; } /// Reads signature of foreign type unit TU. TU is 0-based. @@ -451,7 +451,7 @@ public: return Abbrevs; } - Expected<Entry> getEntry(uint32_t *Offset) const; + Expected<Entry> getEntry(uint64_t *Offset) const; /// Look up all entries in this Name Index matching \c Key. iterator_range<ValueIterator> equal_range(StringRef Key) const; @@ -460,8 +460,8 @@ public: NameIterator end() const { return NameIterator(this, getNameCount() + 1); } Error extract(); - uint32_t getUnitOffset() const { return Base; } - uint32_t getNextUnitOffset() const { return Base + 4 + Hdr.UnitLength; } + uint64_t getUnitOffset() const { return Base; } + uint64_t getNextUnitOffset() const { return Base + 4 + Hdr.UnitLength; } void dump(ScopedPrinter &W) const; friend class DWARFDebugNames; @@ -479,12 +479,12 @@ public: bool IsLocal; Optional<Entry> CurrentEntry; - unsigned DataOffset = 0; ///< Offset into the section. + uint64_t DataOffset = 0; ///< Offset into the section. std::string Key; ///< The Key we are searching for. Optional<uint32_t> Hash; ///< Hash of Key, if it has been computed. bool getEntryAtCurrentOffset(); - Optional<uint32_t> findEntryOffsetInCurrentIndex(); + Optional<uint64_t> findEntryOffsetInCurrentIndex(); bool findInCurrentIndex(); void searchFromStartOfCurrentIndex(); void next(); @@ -572,7 +572,7 @@ public: private: SmallVector<NameIndex, 0> NameIndices; - DenseMap<uint32_t, const NameIndex *> CUToNameIndex; + DenseMap<uint64_t, const NameIndex *> CUToNameIndex; public: DWARFDebugNames(const DWARFDataExtractor &AccelSection, @@ -591,7 +591,7 @@ public: /// Return the Name Index covering the compile unit at CUOffset, or nullptr if /// there is no Name Index covering that unit. - const NameIndex *getCUNameIndex(uint32_t CUOffset); + const NameIndex *getCUNameIndex(uint64_t CUOffset); }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h index c8ad19ad6bf6..dfc778346dbe 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h @@ -23,7 +23,7 @@ namespace llvm { /// attributes in a DWARFDie. struct DWARFAttribute { /// The debug info/types offset for this attribute. - uint32_t Offset = 0; + uint64_t Offset = 0; /// The debug info/types section byte size of the data for this attribute. uint32_t ByteSize = 0; /// The attribute enumeration of this attribute. diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h index 23cf21c3523f..fae163622edb 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -225,10 +225,10 @@ public: DWARFCompileUnit *getDWOCompileUnitForHash(uint64_t Hash); /// Return the compile unit that includes an offset (relative to .debug_info). - DWARFCompileUnit *getCompileUnitForOffset(uint32_t Offset); + DWARFCompileUnit *getCompileUnitForOffset(uint64_t Offset); /// Get a DIE given an exact offset. - DWARFDie getDIEForOffset(uint32_t Offset); + DWARFDie getDIEForOffset(uint64_t Offset); unsigned getMaxVersion() { // Ensure info units have been parsed to discover MaxVersion @@ -301,10 +301,10 @@ public: std::function<void(Error)> RecoverableErrorCallback); DataExtractor getStringExtractor() const { - return DataExtractor(DObj->getStringSection(), false, 0); + return DataExtractor(DObj->getStrSection(), false, 0); } DataExtractor getLineStringExtractor() const { - return DataExtractor(DObj->getLineStringSection(), false, 0); + return DataExtractor(DObj->getLineStrSection(), false, 0); } /// Wraps the returned DIEs for a given address. diff --git a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h index 7c2a159b71fa..980724c525d2 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h @@ -35,20 +35,25 @@ public: /// Extracts a value and applies a relocation to the result if /// one exists for the given offset. - uint64_t getRelocatedValue(uint32_t Size, uint32_t *Off, - uint64_t *SectionIndex = nullptr) const; + uint64_t getRelocatedValue(uint32_t Size, uint64_t *Off, + uint64_t *SectionIndex = nullptr, + Error *Err = nullptr) const; /// Extracts an address-sized value and applies a relocation to the result if /// one exists for the given offset. - uint64_t getRelocatedAddress(uint32_t *Off, uint64_t *SecIx = nullptr) const { + uint64_t getRelocatedAddress(uint64_t *Off, uint64_t *SecIx = nullptr) const { return getRelocatedValue(getAddressSize(), Off, SecIx); } + uint64_t getRelocatedAddress(Cursor &C, uint64_t *SecIx = nullptr) const { + return getRelocatedValue(getAddressSize(), &getOffset(C), SecIx, + &getError(C)); + } /// Extracts a DWARF-encoded pointer in \p Offset using \p Encoding. /// There is a DWARF encoding that uses a PC-relative adjustment. /// For these values, \p AbsPosOffset is used to fix them, which should /// reflect the absolute address of this pointer. - Optional<uint64_t> getEncodedPointer(uint32_t *Offset, uint8_t Encoding, + Optional<uint64_t> getEncodedPointer(uint64_t *Offset, uint8_t Encoding, uint64_t AbsPosOffset = 0) const; size_t size() const { return Section == nullptr ? 0 : Section->Data.size(); } diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h index 28fd8484b4a9..1398e16252a9 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h @@ -20,7 +20,7 @@ namespace llvm { class raw_ostream; class DWARFAbbreviationDeclarationSet { - uint32_t Offset; + uint64_t Offset; /// Code of the first abbreviation, if all abbreviations in the set have /// consecutive codes. UINT32_MAX otherwise. uint32_t FirstAbbrCode; @@ -32,9 +32,9 @@ class DWARFAbbreviationDeclarationSet { public: DWARFAbbreviationDeclarationSet(); - uint32_t getOffset() const { return Offset; } + uint64_t getOffset() const { return Offset; } void dump(raw_ostream &OS) const; - bool extract(DataExtractor Data, uint32_t *OffsetPtr); + bool extract(DataExtractor Data, uint64_t *OffsetPtr); const DWARFAbbreviationDeclaration * getAbbreviationDeclaration(uint32_t AbbrCode) const; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h index a98bf282fe7c..4539b9c9d581 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h @@ -45,7 +45,7 @@ public: private: dwarf::DwarfFormat Format; - uint32_t HeaderOffset; + uint64_t HeaderOffset; Header HeaderData; uint32_t DataSize = 0; std::vector<uint64_t> Addrs; @@ -54,11 +54,11 @@ public: void clear(); /// Extract an entire table, including all addresses. - Error extract(DWARFDataExtractor Data, uint32_t *OffsetPtr, + Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr, uint16_t Version, uint8_t AddrSize, std::function<void(Error)> WarnCallback); - uint32_t getHeaderOffset() const { return HeaderOffset; } + uint64_t getHeaderOffset() const { return HeaderOffset; } uint8_t getAddrSize() const { return HeaderData.AddrSize; } void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) const; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h index 5b6c578bc3bf..ebe4ad6e24dd 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h @@ -49,7 +49,7 @@ private: using DescriptorColl = std::vector<Descriptor>; using desc_iterator_range = iterator_range<DescriptorColl::const_iterator>; - uint32_t Offset; + uint64_t Offset; Header HeaderData; DescriptorColl ArangeDescriptors; @@ -57,7 +57,7 @@ public: DWARFDebugArangeSet() { clear(); } void clear(); - bool extract(DataExtractor data, uint32_t *offset_ptr); + bool extract(DataExtractor data, uint64_t *offset_ptr); void dump(raw_ostream &OS) const; uint32_t getCompileUnitDIEOffset() const { return HeaderData.CuOffset; } diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h index 03223fbc80a9..172f1d2c9dbe 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h @@ -28,7 +28,7 @@ private: void extract(DataExtractor DebugArangesData); /// Call appendRange multiple times and then call construct. - void appendRange(uint32_t CUOffset, uint64_t LowPC, uint64_t HighPC); + void appendRange(uint64_t CUOffset, uint64_t LowPC, uint64_t HighPC); void construct(); struct Range { @@ -60,10 +60,10 @@ private: struct RangeEndpoint { uint64_t Address; - uint32_t CUOffset; + uint64_t CUOffset; bool IsRangeStart; - RangeEndpoint(uint64_t Address, uint32_t CUOffset, bool IsRangeStart) + RangeEndpoint(uint64_t Address, uint64_t CUOffset, bool IsRangeStart) : Address(Address), CUOffset(CUOffset), IsRangeStart(IsRangeStart) {} bool operator<(const RangeEndpoint &Other) const { @@ -76,7 +76,7 @@ private: std::vector<RangeEndpoint> Endpoints; RangeColl Aranges; - DenseSet<uint32_t> ParsedCUOffsets; + DenseSet<uint64_t> ParsedCUOffsets; }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index d960f4bc9b1c..c6539df0d756 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -69,7 +69,7 @@ public: /// starting at *Offset and ending at EndOffset. *Offset is updated /// to EndOffset upon successful parsing, or indicates the offset /// where a problem occurred in case an error is returned. - Error parse(DataExtractor Data, uint32_t *Offset, uint32_t EndOffset); + Error parse(DWARFDataExtractor Data, uint64_t *Offset, uint64_t EndOffset); void dump(raw_ostream &OS, const MCRegisterInfo *MRI, bool IsEH, unsigned IndentLevel = 1) const; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h index f50063b24370..ded960337ec6 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h @@ -22,7 +22,7 @@ class DWARFUnit; /// DWARFDebugInfoEntry - A DIE with only the minimum required data. class DWARFDebugInfoEntry { /// Offset within the .debug_info of the start of this entry. - uint32_t Offset = 0; + uint64_t Offset = 0; /// The integer depth of this DIE within the compile unit DIEs where the /// compile/type unit DIE has a depth of zero. @@ -36,14 +36,14 @@ public: /// Extracts a debug info entry, which is a child of a given unit, /// starting at a given offset. If DIE can't be extracted, returns false and /// doesn't change OffsetPtr. - bool extractFast(const DWARFUnit &U, uint32_t *OffsetPtr); + bool extractFast(const DWARFUnit &U, uint64_t *OffsetPtr); /// High performance extraction should use this call. - bool extractFast(const DWARFUnit &U, uint32_t *OffsetPtr, - const DWARFDataExtractor &DebugInfoData, uint32_t UEndOffset, + bool extractFast(const DWARFUnit &U, uint64_t *OffsetPtr, + const DWARFDataExtractor &DebugInfoData, uint64_t UEndOffset, uint32_t Depth); - uint32_t getOffset() const { return Offset; } + uint64_t getOffset() const { return Offset; } uint32_t getDepth() const { return Depth; } dwarf::Tag getTag() const { diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index e7425c192373..c2be8304ad84 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -18,6 +18,7 @@ #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h" #include "llvm/Support/MD5.h" +#include "llvm/Support/Path.h" #include <cstdint> #include <map> #include <string> @@ -128,13 +129,15 @@ public: bool hasFileAtIndex(uint64_t FileIndex) const; - bool getFileNameByIndex(uint64_t FileIndex, StringRef CompDir, - DILineInfoSpecifier::FileLineInfoKind Kind, - std::string &Result) const; + bool + getFileNameByIndex(uint64_t FileIndex, StringRef CompDir, + DILineInfoSpecifier::FileLineInfoKind Kind, + std::string &Result, + sys::path::Style Style = sys::path::Style::native) const; void clear(); void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const; - Error parse(const DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, + Error parse(const DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr, const DWARFContext &Ctx, const DWARFUnit *U = nullptr); }; @@ -278,7 +281,7 @@ public: /// Parse prologue and all rows. Error parse( - DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, + DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr, const DWARFContext &Ctx, const DWARFUnit *U, std::function<void(Error)> RecoverableErrorCallback, raw_ostream *OS = nullptr); @@ -305,9 +308,9 @@ public: std::vector<uint32_t> &Result) const; }; - const LineTable *getLineTable(uint32_t Offset) const; + const LineTable *getLineTable(uint64_t Offset) const; Expected<const LineTable *> getOrParseLineTable( - DWARFDataExtractor &DebugLineData, uint32_t Offset, + DWARFDataExtractor &DebugLineData, uint64_t Offset, const DWARFContext &Ctx, const DWARFUnit *U, std::function<void(Error)> RecoverableErrorCallback); @@ -350,17 +353,17 @@ public: bool done() const { return Done; } /// Get the offset the parser has reached. - uint32_t getOffset() const { return Offset; } + uint64_t getOffset() const { return Offset; } private: - DWARFUnit *prepareToParse(uint32_t Offset); - void moveToNextTable(uint32_t OldOffset, const Prologue &P); + DWARFUnit *prepareToParse(uint64_t Offset); + void moveToNextTable(uint64_t OldOffset, const Prologue &P); LineToUnitMap LineToUnit; DWARFDataExtractor &DebugLineData; const DWARFContext &Context; - uint32_t Offset = 0; + uint64_t Offset = 0; bool Done = false; }; @@ -377,7 +380,7 @@ private: struct Sequence Sequence; }; - using LineTableMapTy = std::map<uint32_t, LineTable>; + using LineTableMapTy = std::map<uint64_t, LineTable>; using LineTableIter = LineTableMapTy::iterator; using LineTableConstIter = LineTableMapTy::const_iterator; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index cced6048e811..c79d98e34f6e 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -11,6 +11,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include <cstdint> @@ -29,19 +30,20 @@ public: /// The ending address of the instruction range. uint64_t End; /// The location of the variable within the specified range. - SmallVector<char, 4> Loc; + SmallVector<uint8_t, 4> Loc; }; /// A list of locations that contain one variable. struct LocationList { /// The beginning offset where this location list is stored in the debug_loc /// section. - unsigned Offset; + uint64_t Offset; /// All the locations in which the variable is stored. SmallVector<Entry, 2> Entries; /// Dump this list on OS. - void dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, - const MCRegisterInfo *MRI, DWARFUnit *U, uint64_t BaseAddress, + void dump(raw_ostream &OS, uint64_t BaseAddress, bool IsLittleEndian, + unsigned AddressSize, const MCRegisterInfo *MRI, DWARFUnit *U, + DIDumpOptions DumpOpts, unsigned Indent) const; }; @@ -58,7 +60,7 @@ private: public: /// Print the location lists found within the debug_loc section. - void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, + void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, DIDumpOptions DumpOpts, Optional<uint64_t> Offset) const; /// Parse the debug_loc section accessible via the 'data' parameter using the @@ -68,25 +70,29 @@ public: /// Return the location list at the given offset or nullptr. LocationList const *getLocationListAtOffset(uint64_t Offset) const; - Optional<LocationList> parseOneLocationList(DWARFDataExtractor Data, - uint32_t *Offset); + Expected<LocationList> + parseOneLocationList(const DWARFDataExtractor &Data, uint64_t *Offset); }; class DWARFDebugLoclists { public: struct Entry { uint8_t Kind; + uint64_t Offset; uint64_t Value0; uint64_t Value1; - SmallVector<char, 4> Loc; + SmallVector<uint8_t, 4> Loc; + void dump(raw_ostream &OS, uint64_t &BaseAddr, bool IsLittleEndian, + unsigned AddressSize, const MCRegisterInfo *MRI, DWARFUnit *U, + DIDumpOptions DumpOpts, unsigned Indent, size_t MaxEncodingStringLength) const; }; struct LocationList { - unsigned Offset; + uint64_t Offset; SmallVector<Entry, 2> Entries; void dump(raw_ostream &OS, uint64_t BaseAddr, bool IsLittleEndian, unsigned AddressSize, const MCRegisterInfo *RegInfo, - DWARFUnit *U, unsigned Indent) const; + DWARFUnit *U, DIDumpOptions DumpOpts, unsigned Indent) const; }; private: @@ -99,15 +105,16 @@ private: bool IsLittleEndian; public: - void parse(DataExtractor data, unsigned Version); + void parse(DataExtractor data, uint64_t Offset, uint64_t EndOffset, uint16_t Version); void dump(raw_ostream &OS, uint64_t BaseAddr, const MCRegisterInfo *RegInfo, - Optional<uint64_t> Offset) const; + DIDumpOptions DumpOpts, Optional<uint64_t> Offset) const; /// Return the location list at the given offset or nullptr. LocationList const *getLocationListAtOffset(uint64_t Offset) const; - static Optional<LocationList> - parseOneLocationList(DataExtractor Data, unsigned *Offset, unsigned Version); + static Expected<LocationList> parseOneLocationList(const DataExtractor &Data, + uint64_t *Offset, + unsigned Version); }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h index 99e91ca90319..ae57306b90e1 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h @@ -25,7 +25,7 @@ class DWARFDebugPubTable { public: struct Entry { /// Section offset from the beginning of the compilation unit. - uint32_t SecOffset; + uint64_t SecOffset; /// An entry of the various gnu_pub* debug sections. dwarf::PubIndexEntryDescriptor Descriptor; @@ -50,7 +50,7 @@ public: /// The offset from the beginning of the .debug_info section of the /// compilation unit header referenced by the set. - uint32_t Offset; + uint64_t Offset; /// The size in bytes of the contents of the .debug_info section generated /// to represent that compilation unit. diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h index a66f60292343..2f72c642a2d5 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h @@ -53,14 +53,13 @@ public: assert(AddressSize == 4 || AddressSize == 8); if (AddressSize == 4) return StartAddress == -1U; - else - return StartAddress == -1ULL; + return StartAddress == -1ULL; } }; private: /// Offset in .debug_ranges section. - uint32_t Offset; + uint64_t Offset; uint8_t AddressSize; std::vector<RangeListEntry> Entries; @@ -69,7 +68,7 @@ public: void clear(); void dump(raw_ostream &OS) const; - Error extract(const DWARFDataExtractor &data, uint32_t *offset_ptr); + Error extract(const DWARFDataExtractor &data, uint64_t *offset_ptr); const std::vector<RangeListEntry> &getEntries() { return Entries; } /// getAbsoluteRanges - Returns absolute address ranges defined by this range diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h index 167ddde3ec3d..952c41e188c7 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h @@ -34,7 +34,7 @@ struct RangeListEntry : public DWARFListEntryBase { uint64_t Value0; uint64_t Value1; - Error extract(DWARFDataExtractor Data, uint32_t End, uint32_t *OffsetPtr); + Error extract(DWARFDataExtractor Data, uint64_t End, uint64_t *OffsetPtr); void dump(raw_ostream &OS, uint8_t AddrSize, uint8_t MaxEncodingStringLength, uint64_t &CurrentBase, DIDumpOptions DumpOpts, llvm::function_ref<Optional<object::SectionedAddress>(uint32_t)> diff --git a/include/llvm/DebugInfo/DWARF/DWARFDie.h b/include/llvm/DebugInfo/DWARF/DWARFDie.h index 21e68f983bb3..f7f08b4a499d 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -63,7 +63,7 @@ public: /// Get the absolute offset into the debug info or types section. /// /// \returns the DIE offset or -1U if invalid. - uint32_t getOffset() const { + uint64_t getOffset() const { assert(isValid() && "must check validity prior to calling"); return Die->getOffset(); } diff --git a/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/include/llvm/DebugInfo/DWARF/DWARFExpression.h index f066dd58d606..456d9df957ad 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFExpression.h +++ b/include/llvm/DebugInfo/DWARF/DWARFExpression.h @@ -77,18 +77,18 @@ public: uint8_t Opcode; ///< The Op Opcode, DW_OP_<something>. Description Desc; bool Error; - uint32_t EndOffset; + uint64_t EndOffset; uint64_t Operands[2]; - uint32_t OperandEndOffsets[2]; + uint64_t OperandEndOffsets[2]; public: Description &getDescription() { return Desc; } uint8_t getCode() { return Opcode; } uint64_t getRawOperand(unsigned Idx) { return Operands[Idx]; } - uint32_t getOperandEndOffset(unsigned Idx) { return OperandEndOffsets[Idx]; } - uint32_t getEndOffset() { return EndOffset; } + uint64_t getOperandEndOffset(unsigned Idx) { return OperandEndOffsets[Idx]; } + uint64_t getEndOffset() { return EndOffset; } bool extract(DataExtractor Data, uint16_t Version, uint8_t AddressSize, - uint32_t Offset); + uint64_t Offset); bool isError() { return Error; } bool print(raw_ostream &OS, const DWARFExpression *Expr, const MCRegisterInfo *RegInfo, DWARFUnit *U, bool isEH); @@ -101,9 +101,9 @@ public: Operation> { friend class DWARFExpression; const DWARFExpression *Expr; - uint32_t Offset; + uint64_t Offset; Operation Op; - iterator(const DWARFExpression *Expr, uint32_t Offset) + iterator(const DWARFExpression *Expr, uint64_t Offset) : Expr(Expr), Offset(Offset) { Op.Error = Offset >= Expr->Data.getData().size() || diff --git a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 731e71ed9eae..6fec6fcb6b34 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -70,7 +70,7 @@ public: static DWARFFormValue createFromBlockValue(dwarf::Form F, ArrayRef<uint8_t> D); static DWARFFormValue createFromUnit(dwarf::Form F, const DWARFUnit *Unit, - uint32_t *OffsetPtr); + uint64_t *OffsetPtr); dwarf::Form getForm() const { return Form; } uint64_t getRawUValue() const { return Value.uval; } @@ -87,12 +87,12 @@ public: /// in \p FormParams is needed to interpret some forms. The optional /// \p Context and \p Unit allows extracting information if the form refers /// to other sections (e.g., .debug_str). - bool extractValue(const DWARFDataExtractor &Data, uint32_t *OffsetPtr, + bool extractValue(const DWARFDataExtractor &Data, uint64_t *OffsetPtr, dwarf::FormParams FormParams, const DWARFContext *Context = nullptr, const DWARFUnit *Unit = nullptr); - bool extractValue(const DWARFDataExtractor &Data, uint32_t *OffsetPtr, + bool extractValue(const DWARFDataExtractor &Data, uint64_t *OffsetPtr, dwarf::FormParams FormParams, const DWARFUnit *U) { return extractValue(Data, OffsetPtr, FormParams, nullptr, U); } @@ -128,7 +128,7 @@ public: /// \param OffsetPtr A reference to the offset that will be updated. /// \param Params DWARF parameters to help interpret forms. /// \returns true on success, false if the form was not skipped. - bool skipValue(DataExtractor DebugInfoData, uint32_t *OffsetPtr, + bool skipValue(DataExtractor DebugInfoData, uint64_t *OffsetPtr, const dwarf::FormParams Params) const { return DWARFFormValue::skipValue(Form, DebugInfoData, OffsetPtr, Params); } @@ -144,7 +144,7 @@ public: /// \param FormParams DWARF parameters to help interpret forms. /// \returns true on success, false if the form was not skipped. static bool skipValue(dwarf::Form Form, DataExtractor DebugInfoData, - uint32_t *OffsetPtr, + uint64_t *OffsetPtr, const dwarf::FormParams FormParams); private: diff --git a/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/include/llvm/DebugInfo/DWARF/DWARFListTable.h index a1ea69b040f0..496fdb2477f9 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFListTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFListTable.h @@ -26,7 +26,7 @@ namespace llvm { /// entries. struct DWARFListEntryBase { /// The offset at which the entry is located in the section. - uint32_t Offset; + uint64_t Offset; /// The DWARF encoding (DW_RLE_* or DW_LLE_*). uint8_t EntryKind; /// The index of the section this entry belongs to. @@ -46,8 +46,8 @@ public: const ListEntries &getEntries() const { return Entries; } bool empty() const { return Entries.empty(); } void clear() { Entries.clear(); } - Error extract(DWARFDataExtractor Data, uint32_t HeaderOffset, uint32_t End, - uint32_t *OffsetPtr, StringRef SectionName, + Error extract(DWARFDataExtractor Data, uint64_t HeaderOffset, uint64_t End, + uint64_t *OffsetPtr, StringRef SectionName, StringRef ListStringName); }; @@ -57,7 +57,7 @@ class DWARFListTableHeader { struct Header { /// The total length of the entries for this table, not including the length /// field itself. - uint32_t Length = 0; + uint64_t Length = 0; /// The DWARF version number. uint16_t Version; /// The size in bytes of an address on the target architecture. For @@ -75,12 +75,12 @@ class DWARFListTableHeader { /// The offset table, which contains offsets to the individual list entries. /// It is used by forms such as DW_FORM_rnglistx. /// FIXME: Generate the table and use the appropriate forms. - std::vector<uint32_t> Offsets; + std::vector<uint64_t> Offsets; /// The table's format, either DWARF32 or DWARF64. dwarf::DwarfFormat Format; /// The offset at which the header (and hence the table) is located within /// its section. - uint32_t HeaderOffset; + uint64_t HeaderOffset; /// The name of the section the list is located in. StringRef SectionName; /// A characterization of the list for dumping purposes, e.g. "range" or @@ -95,28 +95,40 @@ public: HeaderData = {}; Offsets.clear(); } - uint32_t getHeaderOffset() const { return HeaderOffset; } + uint64_t getHeaderOffset() const { return HeaderOffset; } uint8_t getAddrSize() const { return HeaderData.AddrSize; } - uint32_t getLength() const { return HeaderData.Length; } + uint64_t getLength() const { return HeaderData.Length; } uint16_t getVersion() const { return HeaderData.Version; } StringRef getSectionName() const { return SectionName; } StringRef getListTypeString() const { return ListTypeString; } dwarf::DwarfFormat getFormat() const { return Format; } + /// Return the size of the table header including the length but not including + /// the offsets. + static uint8_t getHeaderSize(dwarf::DwarfFormat Format) { + switch (Format) { + case dwarf::DwarfFormat::DWARF32: + return 12; + case dwarf::DwarfFormat::DWARF64: + return 20; + } + llvm_unreachable("Invalid DWARF format (expected DWARF32 or DWARF64"); + } + void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) const; - Optional<uint32_t> getOffsetEntry(uint32_t Index) const { + Optional<uint64_t> getOffsetEntry(uint32_t Index) const { if (Index < Offsets.size()) return Offsets[Index]; return None; } /// Extract the table header and the array of offsets. - Error extract(DWARFDataExtractor Data, uint32_t *OffsetPtr); + Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr); /// Returns the length of the table, including the length field, or 0 if the /// length has not been determined (e.g. because the table has not yet been /// parsed, or there was a problem in parsing). - uint32_t length() const; + uint64_t length() const; }; /// A class representing a table of lists as specified in the DWARF v5 @@ -128,7 +140,7 @@ template <typename DWARFListType> class DWARFListTableBase { DWARFListTableHeader Header; /// A mapping between file offsets and lists. It is used to find a particular /// list based on an offset (obtained from DW_AT_ranges, for example). - std::map<uint32_t, DWARFListType> ListMap; + std::map<uint64_t, DWARFListType> ListMap; /// This string is displayed as a heading before the list is dumped /// (e.g. "ranges:"). StringRef HeaderString; @@ -144,17 +156,18 @@ public: ListMap.clear(); } /// Extract the table header and the array of offsets. - Error extractHeaderAndOffsets(DWARFDataExtractor Data, uint32_t *OffsetPtr) { + Error extractHeaderAndOffsets(DWARFDataExtractor Data, uint64_t *OffsetPtr) { return Header.extract(Data, OffsetPtr); } /// Extract an entire table, including all list entries. - Error extract(DWARFDataExtractor Data, uint32_t *OffsetPtr); + Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr); /// Look up a list based on a given offset. Extract it and enter it into the /// list map if necessary. - Expected<DWARFListType> findList(DWARFDataExtractor Data, uint32_t Offset); + Expected<DWARFListType> findList(DWARFDataExtractor Data, uint64_t Offset); - uint32_t getHeaderOffset() const { return Header.getHeaderOffset(); } + uint64_t getHeaderOffset() const { return Header.getHeaderOffset(); } uint8_t getAddrSize() const { return Header.getAddrSize(); } + dwarf::DwarfFormat getFormat() const { return Header.getFormat(); } void dump(raw_ostream &OS, llvm::function_ref<Optional<object::SectionedAddress>(uint32_t)> @@ -162,37 +175,31 @@ public: DIDumpOptions DumpOpts = {}) const; /// Return the contents of the offset entry designated by a given index. - Optional<uint32_t> getOffsetEntry(uint32_t Index) const { + Optional<uint64_t> getOffsetEntry(uint32_t Index) const { return Header.getOffsetEntry(Index); } /// Return the size of the table header including the length but not including /// the offsets. This is dependent on the table format, which is unambiguously /// derived from parsing the table. uint8_t getHeaderSize() const { - switch (Header.getFormat()) { - case dwarf::DwarfFormat::DWARF32: - return 12; - case dwarf::DwarfFormat::DWARF64: - return 20; - } - llvm_unreachable("Invalid DWARF format (expected DWARF32 or DWARF64"); + return DWARFListTableHeader::getHeaderSize(getFormat()); } - uint32_t length() { return Header.length(); } + uint64_t length() { return Header.length(); } }; template <typename DWARFListType> Error DWARFListTableBase<DWARFListType>::extract(DWARFDataExtractor Data, - uint32_t *OffsetPtr) { + uint64_t *OffsetPtr) { clear(); if (Error E = extractHeaderAndOffsets(Data, OffsetPtr)) return E; Data.setAddressSize(Header.getAddrSize()); - uint32_t End = getHeaderOffset() + Header.length(); + uint64_t End = getHeaderOffset() + Header.length(); while (*OffsetPtr < End) { DWARFListType CurrentList; - uint32_t Off = *OffsetPtr; + uint64_t Off = *OffsetPtr; if (Error E = CurrentList.extract(Data, getHeaderOffset(), End, OffsetPtr, Header.getSectionName(), Header.getListTypeString())) @@ -208,13 +215,13 @@ Error DWARFListTableBase<DWARFListType>::extract(DWARFDataExtractor Data, template <typename ListEntryType> Error DWARFListType<ListEntryType>::extract(DWARFDataExtractor Data, - uint32_t HeaderOffset, uint32_t End, - uint32_t *OffsetPtr, + uint64_t HeaderOffset, uint64_t End, + uint64_t *OffsetPtr, StringRef SectionName, StringRef ListTypeString) { if (*OffsetPtr < HeaderOffset || *OffsetPtr >= End) return createStringError(errc::invalid_argument, - "invalid %s list offset 0x%" PRIx32, + "invalid %s list offset 0x%" PRIx64, ListTypeString.data(), *OffsetPtr); Entries.clear(); while (*OffsetPtr < End) { @@ -227,7 +234,7 @@ Error DWARFListType<ListEntryType>::extract(DWARFDataExtractor Data, } return createStringError(errc::illegal_byte_sequence, "no end of list marker detected at end of %s table " - "starting at offset 0x%" PRIx32, + "starting at offset 0x%" PRIx64, SectionName.data(), HeaderOffset); } @@ -261,15 +268,15 @@ void DWARFListTableBase<DWARFListType>::dump( template <typename DWARFListType> Expected<DWARFListType> DWARFListTableBase<DWARFListType>::findList(DWARFDataExtractor Data, - uint32_t Offset) { + uint64_t Offset) { auto Entry = ListMap.find(Offset); if (Entry != ListMap.end()) return Entry->second; // Extract the list from the section and enter it into the list map. DWARFListType List; - uint32_t End = getHeaderOffset() + Header.length(); - uint32_t StartingOffset = Offset; + uint64_t End = getHeaderOffset() + Header.length(); + uint64_t StartingOffset = Offset; if (Error E = List.extract(Data, getHeaderOffset(), End, &Offset, Header.getSectionName(), Header.getListTypeString())) diff --git a/include/llvm/DebugInfo/DWARF/DWARFObject.h b/include/llvm/DebugInfo/DWARF/DWARFObject.h index 1bba74a25d0e..88fe3f434edc 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFObject.h +++ b/include/llvm/DebugInfo/DWARF/DWARFObject.h @@ -39,20 +39,20 @@ public: virtual StringRef getAbbrevSection() const { return ""; } virtual const DWARFSection &getLocSection() const { return Dummy; } virtual const DWARFSection &getLoclistsSection() const { return Dummy; } - virtual StringRef getARangeSection() const { return ""; } - virtual StringRef getDebugFrameSection() const { return ""; } - virtual StringRef getEHFrameSection() const { return ""; } + virtual StringRef getArangesSection() const { return ""; } + virtual const DWARFSection &getFrameSection() const { return Dummy; } + virtual const DWARFSection &getEHFrameSection() const { return Dummy; } virtual const DWARFSection &getLineSection() const { return Dummy; } - virtual StringRef getLineStringSection() const { return ""; } - virtual StringRef getStringSection() const { return ""; } - virtual const DWARFSection &getRangeSection() const { return Dummy; } + virtual StringRef getLineStrSection() const { return ""; } + virtual StringRef getStrSection() const { return ""; } + virtual const DWARFSection &getRangesSection() const { return Dummy; } virtual const DWARFSection &getRnglistsSection() const { return Dummy; } virtual StringRef getMacinfoSection() const { return ""; } - virtual const DWARFSection &getPubNamesSection() const { return Dummy; } - virtual const DWARFSection &getPubTypesSection() const { return Dummy; } - virtual const DWARFSection &getGnuPubNamesSection() const { return Dummy; } - virtual const DWARFSection &getGnuPubTypesSection() const { return Dummy; } - virtual const DWARFSection &getStringOffsetSection() const { return Dummy; } + virtual const DWARFSection &getPubnamesSection() const { return Dummy; } + virtual const DWARFSection &getPubtypesSection() const { return Dummy; } + virtual const DWARFSection &getGnuPubnamesSection() const { return Dummy; } + virtual const DWARFSection &getGnuPubtypesSection() const { return Dummy; } + virtual const DWARFSection &getStrOffsetsSection() const { return Dummy; } virtual void forEachInfoDWOSections(function_ref<void(const DWARFSection &)> F) const {} virtual void @@ -60,11 +60,11 @@ public: virtual StringRef getAbbrevDWOSection() const { return ""; } virtual const DWARFSection &getLineDWOSection() const { return Dummy; } virtual const DWARFSection &getLocDWOSection() const { return Dummy; } - virtual StringRef getStringDWOSection() const { return ""; } - virtual const DWARFSection &getStringOffsetDWOSection() const { + virtual StringRef getStrDWOSection() const { return ""; } + virtual const DWARFSection &getStrOffsetsDWOSection() const { return Dummy; } - virtual const DWARFSection &getRangeDWOSection() const { return Dummy; } + virtual const DWARFSection &getRangesDWOSection() const { return Dummy; } virtual const DWARFSection &getRnglistsDWOSection() const { return Dummy; } virtual const DWARFSection &getAddrSection() const { return Dummy; } virtual const DWARFSection &getAppleNamesSection() const { return Dummy; } @@ -72,7 +72,7 @@ public: virtual const DWARFSection &getAppleNamespacesSection() const { return Dummy; } - virtual const DWARFSection &getDebugNamesSection() const { return Dummy; } + virtual const DWARFSection &getNamesSection() const { return Dummy; } virtual const DWARFSection &getAppleObjCSection() const { return Dummy; } virtual StringRef getCUIndexSection() const { return ""; } virtual StringRef getGdbIndexSection() const { return ""; } diff --git a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index 90d89375fd35..c95bdcbd8a43 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -34,7 +34,7 @@ public: LS, LE, IsDWO, UnitVector) {} uint64_t getTypeHash() const { return getHeader().getTypeHash(); } - uint32_t getTypeOffset() const { return getHeader().getTypeOffset(); } + uint64_t getTypeOffset() const { return getHeader().getTypeOffset(); } void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) override; // Enable LLVM-style RTTI. diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/include/llvm/DebugInfo/DWARF/DWARFUnit.h index f9f90db31890..51de114a3506 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -45,7 +45,7 @@ class DWARFUnit; /// parse the header before deciding what specific kind of unit to construct. class DWARFUnitHeader { // Offset within section. - uint32_t Offset = 0; + uint64_t Offset = 0; // Version, address size, and DWARF format. dwarf::FormParams FormParams; uint64_t Length = 0; @@ -56,7 +56,7 @@ class DWARFUnitHeader { // For type units only. uint64_t TypeHash = 0; - uint32_t TypeOffset = 0; + uint64_t TypeOffset = 0; // For v5 split or skeleton compile units only. Optional<uint64_t> DWOId; @@ -70,10 +70,10 @@ class DWARFUnitHeader { public: /// Parse a unit header from \p debug_info starting at \p offset_ptr. bool extract(DWARFContext &Context, const DWARFDataExtractor &debug_info, - uint32_t *offset_ptr, DWARFSectionKind Kind = DW_SECT_INFO, + uint64_t *offset_ptr, DWARFSectionKind Kind = DW_SECT_INFO, const DWARFUnitIndex *Index = nullptr, const DWARFUnitIndex::Entry *Entry = nullptr); - uint32_t getOffset() const { return Offset; } + uint64_t getOffset() const { return Offset; } const dwarf::FormParams &getFormParams() const { return FormParams; } uint16_t getVersion() const { return FormParams.Version; } dwarf::DwarfFormat getFormat() const { return FormParams.Format; } @@ -91,16 +91,17 @@ public: } const DWARFUnitIndex::Entry *getIndexEntry() const { return IndexEntry; } uint64_t getTypeHash() const { return TypeHash; } - uint32_t getTypeOffset() const { return TypeOffset; } + uint64_t getTypeOffset() const { return TypeOffset; } uint8_t getUnitType() const { return UnitType; } bool isTypeUnit() const { return UnitType == dwarf::DW_UT_type || UnitType == dwarf::DW_UT_split_type; } uint8_t getSize() const { return Size; } - uint32_t getNextUnitOffset() const { - return Offset + Length + - (FormParams.Format == llvm::dwarf::DwarfFormat::DWARF64 ? 4 : 0) + - FormParams.getDwarfOffsetByteSize(); + uint8_t getUnitLengthFieldByteSize() const { + return dwarf::getUnitLengthFieldByteSize(FormParams.Format); + } + uint64_t getNextUnitOffset() const { + return Offset + Length + getUnitLengthFieldByteSize(); } }; @@ -110,7 +111,7 @@ const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context, /// Describe a collection of units. Intended to hold all units either from /// .debug_info and .debug_types, or from .debug_info.dwo and .debug_types.dwo. class DWARFUnitVector final : public SmallVector<std::unique_ptr<DWARFUnit>, 1> { - std::function<std::unique_ptr<DWARFUnit>(uint32_t, DWARFSectionKind, + std::function<std::unique_ptr<DWARFUnit>(uint64_t, DWARFSectionKind, const DWARFSection *, const DWARFUnitIndex::Entry *)> Parser; @@ -121,7 +122,7 @@ public: using iterator = typename UnitVector::iterator; using iterator_range = llvm::iterator_range<typename UnitVector::iterator>; - DWARFUnit *getUnitForOffset(uint32_t Offset) const; + DWARFUnit *getUnitForOffset(uint64_t Offset) const; DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E); /// Read units from a .debug_info or .debug_types section. Calls made @@ -197,7 +198,7 @@ class DWARFUnit { DWARFUnitHeader Header; const DWARFDebugAbbrev *Abbrev; const DWARFSection *RangeSection; - uint32_t RangeSectionBase; + uint64_t RangeSectionBase; /// We either keep track of the location list section or its data, depending /// on whether we are handling a split DWARF section or not. union { @@ -275,7 +276,7 @@ public: const DWARFSection &getInfoSection() const { return InfoSection; } const DWARFSection *getLocSection() const { return LocSection; } StringRef getLocSectionData() const { return LocSectionData; } - uint32_t getOffset() const { return Header.getOffset(); } + uint64_t getOffset() const { return Header.getOffset(); } const dwarf::FormParams &getFormParams() const { return Header.getFormParams(); } @@ -285,10 +286,10 @@ public: uint8_t getDwarfOffsetByteSize() const { return Header.getDwarfOffsetByteSize(); } - uint32_t getLength() const { return Header.getLength(); } + uint64_t getLength() const { return Header.getLength(); } uint8_t getUnitType() const { return Header.getUnitType(); } bool isTypeUnit() const { return Header.isTypeUnit(); } - uint32_t getNextUnitOffset() const { return Header.getNextUnitOffset(); } + uint64_t getNextUnitOffset() const { return Header.getNextUnitOffset(); } const DWARFSection &getLineSection() const { return LineSection; } StringRef getStringSection() const { return StringSection; } const DWARFSection &getStringOffsetSection() const { @@ -303,7 +304,7 @@ public: /// Recursively update address to Die map. void updateAddressDieMap(DWARFDie Die); - void setRangesSection(const DWARFSection *RS, uint32_t Base) { + void setRangesSection(const DWARFSection *RS, uint64_t Base) { RangeSection = RS; RangeSectionBase = Base; } @@ -322,7 +323,7 @@ public: /// .debug_ranges section. If the extraction is unsuccessful, an error /// is returned. Successful extraction requires that the compile unit /// has already been extracted. - Error extractRangeList(uint32_t RangeListOffset, + Error extractRangeList(uint64_t RangeListOffset, DWARFDebugRangeList &RangeList) const; void clear(); @@ -405,7 +406,7 @@ public: /// Return a vector of address ranges resulting from a (possibly encoded) /// range list starting at a given offset in the appropriate ranges section. - Expected<DWARFAddressRangesVector> findRnglistFromOffset(uint32_t Offset); + Expected<DWARFAddressRangesVector> findRnglistFromOffset(uint64_t Offset); /// Return a vector of address ranges retrieved from an encoded range /// list whose offset is found via a table lookup given an index (DWARF v5 @@ -415,7 +416,7 @@ public: /// Return a rangelist's offset based on an index. The index designates /// an entry in the rangelist table's offset array and is supplied by /// DW_FORM_rnglistx. - Optional<uint32_t> getRnglistOffset(uint32_t Index) { + Optional<uint64_t> getRnglistOffset(uint32_t Index) { if (RngListTable) return RngListTable->getOffsetEntry(Index); return None; @@ -470,7 +471,7 @@ public: /// unit's DIE vector. /// /// The unit needs to have its DIEs extracted for this method to work. - DWARFDie getDIEForOffset(uint32_t Offset) { + DWARFDie getDIEForOffset(uint64_t Offset) { extractDIEsIfNeeded(false); assert(!DieArray.empty()); auto It = @@ -495,15 +496,19 @@ public: } virtual void dump(raw_ostream &OS, DIDumpOptions DumpOpts) = 0; + + Error tryExtractDIEsIfNeeded(bool CUDieOnly); + private: /// Size in bytes of the .debug_info data associated with this compile unit. size_t getDebugInfoSize() const { - return Header.getLength() + 4 - getHeaderSize(); + return Header.getLength() + Header.getUnitLengthFieldByteSize() - + getHeaderSize(); } /// extractDIEsIfNeeded - Parses a compile unit and indexes its DIEs if it - /// hasn't already been done. Returns the number of DIEs parsed at this call. - size_t extractDIEsIfNeeded(bool CUDieOnly); + /// hasn't already been done + void extractDIEsIfNeeded(bool CUDieOnly); /// extractDIEsToVector - Appends all parsed DIEs to a vector. void extractDIEsToVector(bool AppendCUDie, bool AppendNonCUDIEs, diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h index fc8c707c512e..684103aac2fc 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h +++ b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -37,7 +37,7 @@ class DWARFUnitIndex { uint32_t NumUnits; uint32_t NumBuckets = 0; - bool parse(DataExtractor IndexData, uint32_t *OffsetPtr); + bool parse(DataExtractor IndexData, uint64_t *OffsetPtr); void dump(raw_ostream &OS) const; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h index f1268f220272..a4a3a11d441b 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h +++ b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h @@ -94,7 +94,7 @@ private: /// A map that tracks all references (converted absolute references) so we /// can verify each reference points to a valid DIE and not an offset that /// lies between to valid DIEs. - std::map<uint64_t, std::set<uint32_t>> ReferenceToDIEOffsets; + std::map<uint64_t, std::set<uint64_t>> ReferenceToDIEOffsets; uint32_t NumDebugLineErrors = 0; // Used to relax some checks that do not currently work portably bool IsObjectFile; @@ -138,7 +138,7 @@ private: /// /// \returns true if the header is verified successfully, false otherwise. bool verifyUnitHeader(const DWARFDataExtractor DebugInfoData, - uint32_t *Offset, unsigned UnitIndex, uint8_t &UnitType, + uint64_t *Offset, unsigned UnitIndex, uint8_t &UnitType, bool &isUnitDWARF64); /// Verifies the header of a unit in a .debug_info or .debug_types section. diff --git a/include/llvm/DebugInfo/GSYM/FileEntry.h b/include/llvm/DebugInfo/GSYM/FileEntry.h index 228b4efa0656..49e7fc9c4291 100644 --- a/include/llvm/DebugInfo/GSYM/FileEntry.h +++ b/include/llvm/DebugInfo/GSYM/FileEntry.h @@ -1,9 +1,8 @@ //===- FileEntry.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/GSYM/FileWriter.h b/include/llvm/DebugInfo/GSYM/FileWriter.h new file mode 100644 index 000000000000..cd568765a4f2 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/FileWriter.h @@ -0,0 +1,124 @@ +//===- FileWriter.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_FILEWRITER_H +#define LLVM_DEBUGINFO_GSYM_FILEWRITER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Endian.h" + +#include <stddef.h> +#include <stdint.h> +#include <sys/types.h> + +namespace llvm { +class raw_pwrite_stream; + +namespace gsym { + +/// A simplified binary data writer class that doesn't require targets, target +/// definitions, architectures, or require any other optional compile time +/// libraries to be enabled via the build process. This class needs the ability +/// to seek to different spots in the binary stream that is produces to fixup +/// offsets and sizes. +class FileWriter { + llvm::raw_pwrite_stream &OS; + llvm::support::endianness ByteOrder; +public: + FileWriter(llvm::raw_pwrite_stream &S, llvm::support::endianness B) + : OS(S), ByteOrder(B) {} + ~FileWriter(); + /// Write a single uint8_t value into the stream at the current file + /// position. + /// + /// \param Value The value to write into the stream. + void writeU8(uint8_t Value); + + /// Write a single uint16_t value into the stream at the current file + /// position. The value will be byte swapped if needed to match the byte + /// order specified during construction. + /// + /// \param Value The value to write into the stream. + void writeU16(uint16_t Value); + + /// Write a single uint32_t value into the stream at the current file + /// position. The value will be byte swapped if needed to match the byte + /// order specified during construction. + /// + /// \param Value The value to write into the stream. + void writeU32(uint32_t Value); + + /// Write a single uint64_t value into the stream at the current file + /// position. The value will be byte swapped if needed to match the byte + /// order specified during construction. + /// + /// \param Value The value to write into the stream. + void writeU64(uint64_t Value); + + /// Write the value into the stream encoded using signed LEB128 at the + /// current file position. + /// + /// \param Value The value to write into the stream. + void writeSLEB(int64_t Value); + + /// Write the value into the stream encoded using unsigned LEB128 at the + /// current file position. + /// + /// \param Value The value to write into the stream. + void writeULEB(uint64_t Value); + + /// Write an array of uint8_t values into the stream at the current file + /// position. + /// + /// \param Data An array of values to write into the stream. + void writeData(llvm::ArrayRef<uint8_t> Data); + + /// Write a NULL terminated C string into the stream at the current file + /// position. The entire contents of Str will be written into the steam at + /// the current file position and then an extra NULL termation byte will be + /// written. It is up to the user to ensure that Str doesn't contain any NULL + /// characters unless the additional NULL characters are desired. + /// + /// \param Str The value to write into the stream. + void writeNullTerminated(llvm::StringRef Str); + + /// Fixup a uint32_t value at the specified offset in the stream. This + /// function will save the current file position, seek to the specified + /// offset, overwrite the data using Value, and then restore the file + /// position to the previous file position. + /// + /// \param Value The value to write into the stream. + /// \param Offset The offset at which to write the Value within the stream. + void fixup32(uint32_t Value, uint64_t Offset); + + /// Pad with zeroes at the current file position until the current file + /// position matches the specified alignment. + /// + /// \param Align An integer speciying the desired alignment. This does not + /// need to be a power of two. + void alignTo(size_t Align); + + /// Return the current offset within the file. + /// + /// \return The unsigned offset from the start of the file of the current + /// file position. + uint64_t tell(); + + llvm::raw_pwrite_stream &get_stream() { + return OS; + } + +private: + FileWriter(const FileWriter &rhs) = delete; + void operator=(const FileWriter &rhs) = delete; +}; + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_FILEWRITER_H diff --git a/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/include/llvm/DebugInfo/GSYM/FunctionInfo.h index eedb1e638fd1..63e18bb2ecd5 100644 --- a/include/llvm/DebugInfo/GSYM/FunctionInfo.h +++ b/include/llvm/DebugInfo/GSYM/FunctionInfo.h @@ -1,17 +1,17 @@ //===- FunctionInfo.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H #define LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H +#include "llvm/ADT/Optional.h" #include "llvm/DebugInfo/GSYM/InlineInfo.h" -#include "llvm/DebugInfo/GSYM/LineEntry.h" +#include "llvm/DebugInfo/GSYM/LineTable.h" #include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/DebugInfo/GSYM/StringTable.h" #include <tuple> @@ -21,41 +21,125 @@ namespace llvm { class raw_ostream; namespace gsym { -/// Function information in GSYM files encodes information for one -/// contiguous address range. The name of the function is encoded as -/// a string table offset and allows multiple functions with the same -/// name to share the name string in the string table. Line tables are -/// stored in a sorted vector of gsym::LineEntry objects and are split -/// into line tables for each function. If a function has a discontiguous -/// range, it will be split into two gsym::FunctionInfo objects. If the -/// function has inline functions, the information will be encoded in -/// the "Inline" member, see gsym::InlineInfo for more information. +/// Function information in GSYM files encodes information for one contiguous +/// address range. If a function has discontiguous address ranges, they will +/// need to be encoded using multiple FunctionInfo objects. +/// +/// ENCODING +/// +/// The function information gets the function start address as an argument +/// to the FunctionInfo::decode(...) function. This information is calculated +/// from the GSYM header and an address offset from the GSYM address offsets +/// table. The encoded FunctionInfo information must be alinged to a 4 byte +/// boundary. +/// +/// The encoded data for a FunctionInfo starts with fixed data that all +/// function info objects have: +/// +/// ENCODING NAME DESCRIPTION +/// ========= =========== ==================================================== +/// uint32_t Size The size in bytes of this function. +/// uint32_t Name The string table offset of the function name. +/// +/// The optional data in a FunctionInfo object follows this fixed information +/// and consists of a stream of tuples that consist of: +/// +/// ENCODING NAME DESCRIPTION +/// ========= =========== ==================================================== +/// uint32_t InfoType An "InfoType" enumeration that describes the type +/// of optional data that is encoded. +/// uint32_t InfoLength The size in bytes of the encoded data that +/// immediately follows this length if this value is +/// greater than zero. +/// uint8_t[] InfoData Encoded bytes that represent the data for the +/// "InfoType". These bytes are only present if +/// "InfoLength" is greater than zero. +/// +/// The "InfoType" is an enumeration: +/// +/// enum InfoType { +/// EndOfList = 0u, +/// LineTableInfo = 1u, +/// InlineInfo = 2u +/// }; +/// +/// This stream of tuples is terminated by a "InfoType" whose value is +/// InfoType::EndOfList and a zero for "InfoLength". This signifies the end of +/// the optional information list. This format allows us to add new optional +/// information data to a FunctionInfo object over time and allows older +/// clients to still parse the format and skip over any data that they don't +/// understand or want to parse. +/// +/// So the function information encoding essientially looks like: +/// +/// struct { +/// uint32_t Size; +/// uint32_t Name; +/// struct { +/// uint32_t InfoType; +/// uint32_t InfoLength; +/// uint8_t InfoData[InfoLength]; +/// }[N]; +/// } +/// +/// Where "N" is the number of tuples. struct FunctionInfo { AddressRange Range; uint32_t Name; ///< String table offset in the string table. - std::vector<gsym::LineEntry> Lines; - InlineInfo Inline; + llvm::Optional<LineTable> OptLineTable; + llvm::Optional<InlineInfo> Inline; FunctionInfo(uint64_t Addr = 0, uint64_t Size = 0, uint32_t N = 0) : Range(Addr, Addr + Size), Name(N) {} + /// Query if a FunctionInfo has rich debug info. + /// + /// \returns A bool that indicates if this object has something else than + /// range and name. When converting information from a symbol table and from + /// debug info, we might end up with multiple FunctionInfo objects for the + /// same range and we need to be able to tell which one is the better object + /// to use. bool hasRichInfo() const { - /// Returns whether we have something else than range and name. When - /// converting information from a symbol table and from debug info, we - /// might end up with multiple FunctionInfo objects for the same range - /// and we need to be able to tell which one is the better object to use. - return !Lines.empty() || Inline.isValid(); + return OptLineTable.hasValue() || Inline.hasValue(); } + /// Query if a FunctionInfo object is valid. + /// + /// Address and size can be zero and there can be no line entries for a + /// symbol so the only indication this entry is valid is if the name is + /// not zero. This can happen when extracting information from symbol + /// tables that do not encode symbol sizes. In that case only the + /// address and name will be filled in. + /// + /// \returns A boolean indicating if this FunctionInfo is valid. bool isValid() const { - /// Address and size can be zero and there can be no line entries for a - /// symbol so the only indication this entry is valid is if the name is - /// not zero. This can happen when extracting information from symbol - /// tables that do not encode symbol sizes. In that case only the - /// address and name will be filled in. return Name != 0; } + /// Decode an object from a binary data stream. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param BaseAddr The FunctionInfo's start address and will be used as the + /// base address when decoding any contained information like the line table + /// and the inline info. + /// + /// \returns An FunctionInfo or an error describing the issue that was + /// encountered during decoding. + static llvm::Expected<FunctionInfo> decode(DataExtractor &Data, + uint64_t BaseAddr); + + /// Encode this object into FileWriter stream. + /// + /// \param O The binary stream to write the data to at the current file + /// position. + /// + /// \returns An error object that indicates failure or the offset of the + /// function info that was successfully written into the stream. + llvm::Expected<uint64_t> encode(FileWriter &O) const; + uint64_t startAddress() const { return Range.Start; } uint64_t endAddress() const { return Range.End; } uint64_t size() const { return Range.size(); } @@ -66,14 +150,14 @@ struct FunctionInfo { void clear() { Range = {0, 0}; Name = 0; - Lines.clear(); - Inline.clear(); + OptLineTable = None; + Inline = None; } }; inline bool operator==(const FunctionInfo &LHS, const FunctionInfo &RHS) { return LHS.Range == RHS.Range && LHS.Name == RHS.Name && - LHS.Lines == RHS.Lines && LHS.Inline == RHS.Inline; + LHS.OptLineTable == RHS.OptLineTable && LHS.Inline == RHS.Inline; } inline bool operator!=(const FunctionInfo &LHS, const FunctionInfo &RHS) { return !(LHS == RHS); @@ -89,14 +173,10 @@ inline bool operator<(const FunctionInfo &LHS, const FunctionInfo &RHS) { return LHS.Range < RHS.Range; // Then sort by inline - if (LHS.Inline.isValid() != RHS.Inline.isValid()) - return RHS.Inline.isValid(); - - // If the number of lines is the same, then compare line table entries - if (LHS.Lines.size() == RHS.Lines.size()) - return LHS.Lines < RHS.Lines; - // Then sort by number of line table entries (more is better) - return LHS.Lines.size() < RHS.Lines.size(); + if (LHS.Inline.hasValue() != RHS.Inline.hasValue()) + return RHS.Inline.hasValue(); + + return LHS.OptLineTable < RHS.OptLineTable; } raw_ostream &operator<<(raw_ostream &OS, const FunctionInfo &R); diff --git a/include/llvm/DebugInfo/GSYM/GsymCreator.h b/include/llvm/DebugInfo/GSYM/GsymCreator.h new file mode 100644 index 000000000000..12c8187132ba --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/GsymCreator.h @@ -0,0 +1,229 @@ +//===- GsymCreator.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_GSYMCREATOR_H +#define LLVM_DEBUGINFO_GSYM_GSYMCREATOR_H + +#include <functional> +#include <memory> +#include <mutex> +#include <string> +#include <thread> + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/GSYM/FileEntry.h" +#include "llvm/DebugInfo/GSYM/FunctionInfo.h" +#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/MC/StringTableBuilder.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Path.h" + +namespace llvm { + +namespace gsym { +class FileWriter; + +/// GsymCreator is used to emit GSYM data to a stand alone file or section +/// within a file. +/// +/// The GsymCreator is designed to be used in 3 stages: +/// - Create FunctionInfo objects and add them +/// - Finalize the GsymCreator object +/// - Save to file or section +/// +/// The first stage involves creating FunctionInfo objects from another source +/// of information like compiler debug info metadata, DWARF or Breakpad files. +/// Any strings in the FunctionInfo or contained information, like InlineInfo +/// or LineTable objects, should get the string table offsets by calling +/// GsymCreator::insertString(...). Any file indexes that are needed should be +/// obtained by calling GsymCreator::insertFile(...). All of the function calls +/// in GsymCreator are thread safe. This allows multiple threads to create and +/// add FunctionInfo objects while parsing debug information. +/// +/// Once all of the FunctionInfo objects have been added, the +/// GsymCreator::finalize(...) must be called prior to saving. This function +/// will sort the FunctionInfo objects, finalize the string table, and do any +/// other passes on the information needed to prepare the information to be +/// saved. +/// +/// Once the object has been finalized, it can be saved to a file or section. +/// +/// ENCODING +/// +/// GSYM files are designed to be memory mapped into a process as shared, read +/// only data, and used as is. +/// +/// The GSYM file format when in a stand alone file consists of: +/// - Header +/// - Address Table +/// - Function Info Offsets +/// - File Table +/// - String Table +/// - Function Info Data +/// +/// HEADER +/// +/// The header is fully described in "llvm/DebugInfo/GSYM/Header.h". +/// +/// ADDRESS TABLE +/// +/// The address table immediately follows the header in the file and consists +/// of Header.NumAddresses address offsets. These offsets are sorted and can be +/// binary searched for efficient lookups. Addresses in the address table are +/// stored as offsets from a 64 bit base address found in Header.BaseAddress. +/// This allows the address table to contain 8, 16, or 32 offsets. This allows +/// the address table to not require full 64 bit addresses for each address. +/// The resulting GSYM size is smaller and causes fewer pages to be touched +/// during address lookups when the address table is smaller. The size of the +/// address offsets in the address table is specified in the header in +/// Header.AddrOffSize. The first offset in the address table is alinged to +/// Header.AddrOffSize alignement to ensure efficient access when loaded into +/// memory. +/// +/// FUNCTION INFO OFFSETS TABLE +/// +/// The function info offsets table immediately follows the address table and +/// consists of Header.NumAddresses 32 bit file offsets: one for each address +/// in the address table. This data is algined to a 4 byte boundary. The +/// offsets in this table are the relative offsets from the start offset of the +/// GSYM header and point to the function info data for each address in the +/// address table. Keeping this data separate from the address table helps to +/// reduce the number of pages that are touched when address lookups occur on a +/// GSYM file. +/// +/// FILE TABLE +/// +/// The file table immediately follows the function info offsets table. The +/// encoding of the FileTable is: +/// +/// struct FileTable { +/// uint32_t Count; +/// FileEntry Files[]; +/// }; +/// +/// The file table starts with a 32 bit count of the number of files that are +/// used in all of the function info, followed by that number of FileEntry +/// structures. The file table is aligned to a 4 byte boundary, Each file in +/// the file table is represented with a FileEntry structure. +/// See "llvm/DebugInfo/GSYM/FileEntry.h" for details. +/// +/// STRING TABLE +/// +/// The string table follows the file table in stand alone GSYM files and +/// contains all strings for everything contained in the GSYM file. Any string +/// data should be added to the string table and any references to strings +/// inside GSYM information must be stored as 32 bit string table offsets into +/// this string table. The string table always starts with an empty string at +/// offset zero and is followed by any strings needed by the GSYM information. +/// The start of the string table is not aligned to any boundary. +/// +/// FUNCTION INFO DATA +/// +/// The function info data is the payload that contains information about the +/// address that is being looked up. It contains all of the encoded +/// FunctionInfo objects. Each encoded FunctionInfo's data is pointed to by an +/// entry in the Function Info Offsets Table. For details on the exact encoding +/// of FunctionInfo objects, see "llvm/DebugInfo/GSYM/FunctionInfo.h". +class GsymCreator { + // Private member variables require Mutex protections + mutable std::recursive_mutex Mutex; + std::vector<FunctionInfo> Funcs; + StringTableBuilder StrTab; + DenseMap<llvm::gsym::FileEntry, uint32_t> FileEntryToIndex; + std::vector<llvm::gsym::FileEntry> Files; + std::vector<uint8_t> UUID; + bool Finalized = false; + +public: + + GsymCreator(); + + /// Save a GSYM file to a stand alone file. + /// + /// \param Path The file path to save the GSYM file to. + /// \param ByteOrder The endianness to use when saving the file. + /// \returns An error object that indicates success or failure of the save. + llvm::Error save(StringRef Path, llvm::support::endianness ByteOrder) const; + + /// Encode a GSYM into the file writer stream at the current position. + /// + /// \param O The stream to save the binary data to + /// \returns An error object that indicates success or failure of the save. + llvm::Error encode(FileWriter &O) const; + + /// Insert a string into the GSYM string table. + /// + /// All strings used by GSYM files must be uniqued by adding them to this + /// string pool and using the returned offset for any string values. + /// + /// \param S The string to insert into the string table. + /// \returns The unique 32 bit offset into the string table. + uint32_t insertString(StringRef S); + + /// Insert a file into this GSYM creator. + /// + /// Inserts a file by adding a FileEntry into the "Files" member variable if + /// the file has not already been added. The file path is split into + /// directory and filename which are both added to the string table. This + /// allows paths to be stored efficiently by reusing the directories that are + /// common between multiple files. + /// + /// \param Path The path to the file to insert. + /// \param Style The path style for the "Path" parameter. + /// \returns The unique file index for the inserted file. + uint32_t insertFile(StringRef Path, + sys::path::Style Style = sys::path::Style::native); + + /// Add a function info to this GSYM creator. + /// + /// All information in the FunctionInfo object must use the + /// GsymCreator::insertString(...) function when creating string table + /// offsets for names and other strings. + /// + /// \param FI The function info object to emplace into our functions list. + void addFunctionInfo(FunctionInfo &&FI); + + /// Finalize the data in the GSYM creator prior to saving the data out. + /// + /// Finalize must be called after all FunctionInfo objects have been added + /// and before GsymCreator::save() is called. + /// + /// \param OS Output stream to report duplicate function infos, overlapping + /// function infos, and function infos that were merged or removed. + /// \returns An error object that indicates success or failure of the + /// finalize. + llvm::Error finalize(llvm::raw_ostream &OS); + + /// Set the UUID value. + /// + /// \param UUIDBytes The new UUID bytes. + void setUUID(llvm::ArrayRef<uint8_t> UUIDBytes) { + UUID.assign(UUIDBytes.begin(), UUIDBytes.end()); + } + + /// Thread safe iteration over all function infos. + /// + /// \param Callback A callback function that will get called with each + /// FunctionInfo. If the callback returns false, stop iterating. + void forEachFunctionInfo( + std::function<bool(FunctionInfo &)> const &Callback); + + /// Thread safe const iteration over all function infos. + /// + /// \param Callback A callback function that will get called with each + /// FunctionInfo. If the callback returns false, stop iterating. + void forEachFunctionInfo( + std::function<bool(const FunctionInfo &)> const &Callback) const; + +}; + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_GSYMCREATOR_H diff --git a/include/llvm/DebugInfo/GSYM/GsymReader.h b/include/llvm/DebugInfo/GSYM/GsymReader.h new file mode 100644 index 000000000000..113bcee9c9a3 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/GsymReader.h @@ -0,0 +1,228 @@ +//===- GsymReader.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_GSYMREADER_H +#define LLVM_DEBUGINFO_GSYM_GSYMREADER_H + + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/GSYM/FileEntry.h" +#include "llvm/DebugInfo/GSYM/FunctionInfo.h" +#include "llvm/DebugInfo/GSYM/Header.h" +#include "llvm/DebugInfo/GSYM/LineEntry.h" +#include "llvm/DebugInfo/GSYM/StringTable.h" +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/ErrorOr.h" + +#include <inttypes.h> +#include <memory> +#include <stdint.h> +#include <string> +#include <vector> + +namespace llvm { +class MemoryBuffer; +class raw_ostream; + +namespace gsym { + +/// GsymReader is used to read GSYM data from a file or buffer. +/// +/// This class is optimized for very quick lookups when the endianness matches +/// the host system. The Header, address table, address info offsets, and file +/// table is designed to be mmap'ed as read only into memory and used without +/// any parsing needed. If the endianness doesn't match, we swap these objects +/// and tables into GsymReader::SwappedData and then point our header and +/// ArrayRefs to this swapped internal data. +/// +/// GsymReader objects must use one of the static functions to create an +/// instance: GsymReader::openFile(...) and GsymReader::copyBuffer(...). + +class GsymReader { + GsymReader(std::unique_ptr<MemoryBuffer> Buffer); + llvm::Error parse(); + + std::unique_ptr<MemoryBuffer> MemBuffer; + StringRef GsymBytes; + llvm::support::endianness Endian; + const Header *Hdr = nullptr; + ArrayRef<uint8_t> AddrOffsets; + ArrayRef<uint32_t> AddrInfoOffsets; + ArrayRef<FileEntry> Files; + StringTable StrTab; + /// When the GSYM file's endianness doesn't match the host system then + /// we must decode all data structures that need to be swapped into + /// local storage and set point the ArrayRef objects above to these swapped + /// copies. + struct SwappedData { + Header Hdr; + std::vector<uint8_t> AddrOffsets; + std::vector<uint32_t> AddrInfoOffsets; + std::vector<FileEntry> Files; + }; + std::unique_ptr<SwappedData> Swap; + +public: + GsymReader(GsymReader &&RHS); + ~GsymReader(); + + /// Construct a GsymReader from a file on disk. + /// + /// \param Path The file path the GSYM file to read. + /// \returns An expected GsymReader that contains the object or an error + /// object that indicates reason for failing to read the GSYM. + static llvm::Expected<GsymReader> openFile(StringRef Path); + + /// Construct a GsymReader from a buffer. + /// + /// \param Bytes A set of bytes that will be copied and owned by the + /// returned object on success. + /// \returns An expected GsymReader that contains the object or an error + /// object that indicates reason for failing to read the GSYM. + static llvm::Expected<GsymReader> copyBuffer(StringRef Bytes); + + /// Access the GSYM header. + /// \returns A native endian version of the GSYM header. + const Header &getHeader() const; + + /// Get the full function info for an address. + /// + /// \param Addr A virtual address from the orignal object file to lookup. + /// \returns An expected FunctionInfo that contains the function info object + /// or an error object that indicates reason for failing to lookup the + /// address, + llvm::Expected<FunctionInfo> getFunctionInfo(uint64_t Addr) const; + + /// Get a string from the string table. + /// + /// \param Offset The string table offset for the string to retrieve. + /// \returns The string from the strin table. + StringRef getString(uint32_t Offset) const { return StrTab[Offset]; } + +protected: + /// Gets an address from the address table. + /// + /// Addresses are stored as offsets frrom the gsym::Header::BaseAddress. + /// + /// \param Index A index into the address table. + /// \returns A resolved virtual address for adddress in the address table + /// or llvm::None if Index is out of bounds. + Optional<uint64_t> getAddress(size_t Index) const; + + /// Get the a file entry for the suppplied file index. + /// + /// Used to convert any file indexes in the FunctionInfo data back into + /// files. This function can be used for iteration, but is more commonly used + /// for random access when doing lookups. + /// + /// \param Index An index into the file table. + /// \returns An optional FileInfo that will be valid if the file index is + /// valid, or llvm::None if the file index is out of bounds, + Optional<FileEntry> getFile(uint32_t Index) const { + if (Index < Files.size()) + return Files[Index]; + return llvm::None; + } + + /// Get an appropriate address info offsets array. + /// + /// The address table in the GSYM file is stored as array of 1, 2, 4 or 8 + /// byte offsets from the The gsym::Header::BaseAddress. The table is stored + /// internally as a array of bytes that are in the correct endianness. When + /// we access this table we must get an array that matches those sizes. This + /// templatized helper function is used when accessing address offsets in the + /// AddrOffsets member variable. + /// + /// \returns An ArrayRef of an appropriate address offset size. + template <class T> ArrayRef<T> + getAddrOffsets() const { + return ArrayRef<T>(reinterpret_cast<const T *>(AddrOffsets.data()), + AddrOffsets.size()/sizeof(T)); + } + + /// Get an appropriate address from the address table. + /// + /// The address table in the GSYM file is stored as array of 1, 2, 4 or 8 + /// byte address offsets from the The gsym::Header::BaseAddress. The table is + /// stored internally as a array of bytes that are in the correct endianness. + /// In order to extract an address from the address table we must access the + /// address offset using the correct size and then add it to the BaseAddress + /// in the header. + /// + /// \param Index An index into the AddrOffsets array. + /// \returns An virtual address that matches the original object file for the + /// address as the specified index, or llvm::None if Index is out of bounds. + template <class T> Optional<uint64_t> + addressForIndex(size_t Index) const { + ArrayRef<T> AIO = getAddrOffsets<T>(); + if (Index < AIO.size()) + return AIO[Index] + Hdr->BaseAddress; + return llvm::None; + } + /// Lookup an address offset in the AddrOffsets table. + /// + /// Given an address offset, look it up using a binary search of the + /// AddrOffsets table. + /// + /// \param AddrOffset An address offset, that has already been computed by + /// subtracting the gsym::Header::BaseAddress. + /// \returns The matching address offset index. This index will be used to + /// extract the FunctionInfo data's offset from the AddrInfoOffsets array. + template <class T> + uint64_t getAddressOffsetIndex(const uint64_t AddrOffset) const { + ArrayRef<T> AIO = getAddrOffsets<T>(); + const auto Begin = AIO.begin(); + const auto End = AIO.end(); + auto Iter = std::lower_bound(Begin, End, AddrOffset); + if (Iter == End || AddrOffset < *Iter) + --Iter; + return std::distance(Begin, Iter); + } + + /// Create a GSYM from a memory buffer. + /// + /// Called by both openFile() and copyBuffer(), this function does all of the + /// work of parsing the GSYM file and returning an error. + /// + /// \param MemBuffer A memory buffer that will transfer ownership into the + /// GsymReader. + /// \returns An expected GsymReader that contains the object or an error + /// object that indicates reason for failing to read the GSYM. + static llvm::Expected<llvm::gsym::GsymReader> + create(std::unique_ptr<MemoryBuffer> &MemBuffer); + + + /// Given an address, find the address index. + /// + /// Binary search the address table and find the matching address index. + /// + /// \param Addr A virtual address that matches the original object file + /// to lookup. + /// \returns An index into the address table. This index can be used to + /// extract the FunctionInfo data's offset from the AddrInfoOffsets array. + /// Returns an error if the address isn't in the GSYM with details of why. + Expected<uint64_t> getAddressIndex(const uint64_t Addr) const; + + /// Given an address index, get the offset for the FunctionInfo. + /// + /// Looking up an address is done by finding the corresponding address + /// index for the address. This index is then used to get the offset of the + /// FunctionInfo data that we will decode using this function. + /// + /// \param Index An index into the address table. + /// \returns An optional GSYM data offset for the offset of the FunctionInfo + /// that needs to be decoded. + Optional<uint64_t> getAddressInfoOffset(size_t Index) const; +}; + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_GSYMREADER_H diff --git a/include/llvm/DebugInfo/GSYM/Header.h b/include/llvm/DebugInfo/GSYM/Header.h new file mode 100644 index 000000000000..6652c59c97a6 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/Header.h @@ -0,0 +1,129 @@ +//===- Header.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_HEADER_H +#define LLVM_DEBUGINFO_GSYM_HEADER_H + +#include "llvm/Support/Error.h" + +#include <cstddef> +#include <cstdint> + +namespace llvm { +class raw_ostream; +class DataExtractor; + +namespace gsym { +class FileWriter; + +constexpr uint32_t GSYM_MAGIC = 0x4753594d; // 'GSYM' +constexpr uint32_t GSYM_CIGAM = 0x4d595347; // 'MYSG' +constexpr uint32_t GSYM_VERSION = 1; +constexpr size_t GSYM_MAX_UUID_SIZE = 20; + +/// The GSYM header. +/// +/// The GSYM header is found at the start of a stand alone GSYM file, or as +/// the first bytes in a section when GSYM is contained in a section of an +/// executable file (ELF, mach-o, COFF). +/// +/// The structure is encoded exactly as it appears in the structure definition +/// with no gaps between members. Alignment should not change from system to +/// system as the members were laid out so that they shouldn't align +/// differently on different architectures. +/// +/// When endianness of the system loading a GSYM file matches, the file can +/// be mmap'ed in and a pointer to the header can be cast to the first bytes +/// of the file (stand alone GSYM file) or section data (GSYM in a section). +/// When endianness is swapped, the Header::decode() function should be used to +/// decode the header. +struct Header { + /// The magic bytes should be set to GSYM_MAGIC. This helps detect if a file + /// is a GSYM file by scanning the first 4 bytes of a file or section. + /// This value might appear byte swapped + uint32_t Magic; + /// The version can number determines how the header is decoded and how each + /// InfoType in FunctionInfo is encoded/decoded. As version numbers increase, + /// "Magic" and "Version" members should always appear at offset zero and 4 + /// respectively to ensure clients figure out if they can parse the format. + uint16_t Version; + /// The size in bytes of each address offset in the address offsets table. + uint8_t AddrOffSize; + /// The size in bytes of the UUID encoded in the "UUID" member. + uint8_t UUIDSize; + /// The 64 bit base address that all address offsets in the address offsets + /// table are relative to. Storing a full 64 bit address allows our address + /// offsets table to be smaller on disk. + uint64_t BaseAddress; + /// The number of addresses stored in the address offsets table. + uint32_t NumAddresses; + /// The file relative offset of the start of the string table for strings + /// contained in the GSYM file. If the GSYM in contained in a stand alone + /// file this will be the file offset of the start of the string table. If + /// the GSYM is contained in a section within an executable file, this can + /// be the offset of the first string used in the GSYM file and can possibly + /// span one or more executable string tables. This allows the strings to + /// share string tables in an ELF or mach-o file. + uint32_t StrtabOffset; + /// The size in bytes of the string table. For a stand alone GSYM file, this + /// will be the exact size in bytes of the string table. When the GSYM data + /// is in a section within an executable file, this size can span one or more + /// sections that contains strings. This allows any strings that are already + /// stored in the executable file to be re-used, and any extra strings could + /// be added to another string table and the string table offset and size + /// can be set to span all needed string tables. + uint32_t StrtabSize; + /// The UUID of the original executable file. This is stored to allow + /// matching a GSYM file to an executable file when symbolication is + /// required. Only the first "UUIDSize" bytes of the UUID are valid. Any + /// bytes in the UUID value that appear after the first UUIDSize bytes should + /// be set to zero. + uint8_t UUID[GSYM_MAX_UUID_SIZE]; + + /// Check if a header is valid and return an error if anything is wrong. + /// + /// This function can be used prior to encoding a header to ensure it is + /// valid, or after decoding a header to ensure it is valid and supported. + /// + /// Check a correctly byte swapped header for errors: + /// - check magic value + /// - check that version number is supported + /// - check that the address offset size is supported + /// - check that the UUID size is valid + /// + /// \returns An error if anything is wrong in the header, or Error::success() + /// if there are no errors. + llvm::Error checkForError() const; + + /// Decode an object from a binary data stream. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \returns A Header or an error describing the issue that was + /// encountered during decoding. + static llvm::Expected<Header> decode(DataExtractor &Data); + + /// Encode this object into FileWriter stream. + /// + /// \param O The binary stream to write the data to at the current file + /// position. + /// + /// \returns An error object that indicates success or failure of the + /// encoding process. + llvm::Error encode(FileWriter &O) const; +}; + +bool operator==(const Header &LHS, const Header &RHS); +raw_ostream &operator<<(raw_ostream &OS, const llvm::gsym::Header &H); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_HEADER_H diff --git a/include/llvm/DebugInfo/GSYM/InlineInfo.h b/include/llvm/DebugInfo/GSYM/InlineInfo.h index 222430622932..48fd9a7c1308 100644 --- a/include/llvm/DebugInfo/GSYM/InlineInfo.h +++ b/include/llvm/DebugInfo/GSYM/InlineInfo.h @@ -1,9 +1,8 @@ //===- InlineInfo.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,6 +11,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/Support/Error.h" #include <stdint.h> #include <vector> @@ -31,6 +31,30 @@ namespace gsym { /// Any clients that encode information will need to ensure the ranges are /// all contined correctly or lookups could fail. Add ranges in these objects /// must be contained in the top level FunctionInfo address ranges as well. +/// +/// ENCODING +/// +/// When saved to disk, the inline info encodes all ranges to be relative to +/// a parent address range. This will be the FunctionInfo's start address if +/// the InlineInfo is directly contained in a FunctionInfo, or a the start +/// address of the containing parent InlineInfo's first "Ranges" member. This +/// allows address ranges to be efficiently encoded using ULEB128 encodings as +/// we encode the offset and size of each range instead of full addresses. This +/// also makes any encoded addresses easy to relocate as we just need to +/// relocate the FunctionInfo's start address. +/// +/// - The AddressRanges member "Ranges" is encoded using an approriate base +/// address as described above. +/// - UINT8 boolean value that specifies if the InlineInfo object has children. +/// - UINT32 string table offset that points to the name of the inline +/// function. +/// - ULEB128 integer that specifies the file of the call site that called +/// this function. +/// - ULEB128 integer that specifies the source line of the call site that +/// called this function. +/// - if this object has children, enocode each child InlineInfo using the +/// the first address range's start address as the base address. +/// struct InlineInfo { uint32_t Name; ///< String table offset in the string table. @@ -62,6 +86,37 @@ struct InlineInfo { /// \returns optional vector of InlineInfo objects that describe the /// inline call stack for a given address, false otherwise. llvm::Optional<InlineArray> getInlineStack(uint64_t Addr) const; + + /// Decode an InlineInfo object from a binary data stream. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the InlineInfo object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param BaseAddr The base address to use when decoding all address ranges. + /// This will be the FunctionInfo's start address if this object is directly + /// contained in a FunctionInfo object, or the start address of the first + /// address range in an InlineInfo object of this object is a child of + /// another InlineInfo object. + /// \returns An InlineInfo or an error describing the issue that was + /// encountered during decoding. + static llvm::Expected<InlineInfo> decode(DataExtractor &Data, + uint64_t BaseAddr); + + /// Encode this InlineInfo object into FileWriter stream. + /// + /// \param O The binary stream to write the data to at the current file + /// position. + /// + /// \param BaseAddr The base address to use when encoding all address ranges. + /// This will be the FunctionInfo's start address if this object is directly + /// contained in a FunctionInfo object, or the start address of the first + /// address range in an InlineInfo object of this object is a child of + /// another InlineInfo object. + /// + /// \returns An error object that indicates success or failure or the + /// encoding process. + llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const; }; inline bool operator==(const InlineInfo &LHS, const InlineInfo &RHS) { diff --git a/include/llvm/DebugInfo/GSYM/LineEntry.h b/include/llvm/DebugInfo/GSYM/LineEntry.h index 6b9380940bd3..aac7c48e067e 100644 --- a/include/llvm/DebugInfo/GSYM/LineEntry.h +++ b/include/llvm/DebugInfo/GSYM/LineEntry.h @@ -1,9 +1,8 @@ //===- LineEntry.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/GSYM/LineTable.h b/include/llvm/DebugInfo/GSYM/LineTable.h new file mode 100644 index 000000000000..3cdbccb08ced --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/LineTable.h @@ -0,0 +1,198 @@ +//===- LineTable.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_LINETABLE_H +#define LLVM_DEBUGINFO_GSYM_LINETABLE_H + +#include "llvm/DebugInfo/GSYM/LineEntry.h" +#include "llvm/Support/Error.h" +#include <cstdint> +#include <vector> + +namespace llvm { +namespace gsym { + +struct FunctionInfo; +class FileWriter; + +/// LineTable class contains deserialized versions of line tables for each +/// function's address ranges. +/// +/// When saved to disk, the line table is encoded using a modified version of +/// the DWARF line tables that only tracks address to source file and line. +/// +/// ENCODING +/// +/// The line table starts with a small prolog that contains the following +/// values: +/// +/// ENCODING NAME DESCRIPTION +/// ======== =========== ==================================================== +/// SLEB MinDelta The min line delta for special opcodes that advance +/// the address and line number. +/// SLEB MaxDelta The max line delta for single byte opcodes that +/// advance the address and line number. +/// ULEB FirstLine The value of the first source line number to +/// initialize the LineEntry with. +/// +/// Once these prolog items are read, we initialize a LineEntry struct with +/// the start address of the function from the FunctionInfo's address range, +/// a default file index of 1, and the line number set to "FirstLine" from +/// the prolog above: +/// +/// LineEntry Row(BaseAddr, 1, FirstLine); +/// +/// The line table state machine is now initialized and ready to be parsed. +/// The stream that follows this encodes the line entries in a compact +/// form. Some opcodes cause "Row" to be modified and some opcodes may also +/// push "Row" onto the end of the "LineTable.Lines" vector. The end result +/// is a vector of LineEntry structs that is sorted in ascending address +/// order. +/// +/// NORMAL OPCODES +/// +/// The opcodes 0 through 3 are normal in opcodes. Their encoding and +/// descriptions are listed below: +/// +/// ENCODING ENUMERATION VALUE DESCRIPTION +/// ======== ================ ===== ======================================== +/// LTOC_EndSequence 0x00 Parsing is done. +/// ULEB LTOC_SetFile 0x01 Row.File = ULEB +/// ULEB LTOC_AdvancePC 0x02 Row.Addr += ULEB, push "Row". +/// SLEB LTOC_AdvanceLine 0x03 Row.Line += SLEB +/// LTOC_FirstSpecial 0x04 First special opcode (see SPECIAL +/// OPCODES below). +/// +/// SPECIAL OPCODES +/// +/// Opcodes LTOC_FirstSpecial through 255 are special opcodes that always +/// increment both the Row.Addr and Row.Line and push "Row" onto the +/// LineEntry.Lines array. They do this by using some of the bits to +/// increment/decrement the source line number, and some of the bits to +/// increment the address. Line numbers can go up or down when making line +/// tables, where addresses always only increase since line tables are sorted +/// by address. +/// +/// In order to calculate the amount to increment the line and address for +/// these special opcodes, we calculate the number of values reserved for the +/// line increment/decrement using the "MinDelta" and "MaxDelta" from the +/// prolog: +/// +/// const int64_t LineRange = MaxDelta - MinDelta + 1; +/// +/// Then we can adjust the opcode to not include any of the normal opcodes: +/// +/// const uint8_t AdjustedOp = Opcode - LTOC_FirstSpecial; +/// +/// And we can calculate the line offset, and address offset: +/// +/// const int64_t LineDelta = MinDelta + (AdjustedOp % LineRange); +/// const uint64_t AddrDelta = (AdjustedOp / LineRange); +/// +/// And use these to modify our "Row": +/// +/// Row.Line += LineDelta; +/// Row.Addr += AddrDelta; +/// +/// And push a row onto the line table: +/// +/// Lines.push_back(Row); +/// +/// This is verify similar to the way that DWARF encodes its line tables. The +/// only difference is the DWARF line tables have more normal opcodes and the +/// "Row" contains more members, like source column number, bools for end of +/// prologue, beginnging of epilogue, is statement and many others. There are +/// also more complex rules that happen for the extra normal opcodes. By +/// leaving these extra opcodes out, we leave more bits for the special +/// opcodes that allows us to encode line tables in fewer bytes than standard +/// DWARF encodings. +/// +/// Opcodes that will push "Row" onto the LineEntry.Lines include the +/// LTOC_AdvancePC opcode and all special opcodes. All other opcodes +/// only modify the current "Row", or cause the line table to end. +class LineTable { + typedef std::vector<gsym::LineEntry> Collection; + Collection Lines; ///< All line entries in the line table. +public: + static LineEntry lookup(DataExtractor &Data, uint64_t BaseAddr, + uint64_t Addr); + + /// Decode an LineTable object from a binary data stream. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the LineTable object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param BaseAddr The base address to use when decoding the line table. + /// This will be the FunctionInfo's start address and will be used to + /// initialize the line table row prior to parsing any opcodes. + /// + /// \returns An LineTable or an error describing the issue that was + /// encountered during decoding. + static llvm::Expected<LineTable> decode(DataExtractor &Data, + uint64_t BaseAddr); + /// Encode this LineTable object into FileWriter stream. + /// + /// \param O The binary stream to write the data to at the current file + /// position. + /// + /// \param BaseAddr The base address to use when decoding the line table. + /// This will be the FunctionInfo's start address. + /// + /// \returns An error object that indicates success or failure or the + /// encoding process. + llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const; + bool empty() const { return Lines.empty(); } + void clear() { Lines.clear(); } + void push(const LineEntry &LE) { + Lines.push_back(LE); + } + size_t isValid() const { + return !Lines.empty(); + } + size_t size() const { + return Lines.size(); + } + LineEntry &get(size_t i) { + assert(i < Lines.size()); + return Lines[i]; + } + const LineEntry &get(size_t i) const { + assert(i < Lines.size()); + return Lines[i]; + } + LineEntry &operator[](size_t i) { + return get(i); + } + const LineEntry &operator[](size_t i) const { + return get(i); + } + bool operator==(const LineTable &RHS) const { + return Lines == RHS.Lines; + } + bool operator!=(const LineTable &RHS) const { + return Lines != RHS.Lines; + } + bool operator<(const LineTable &RHS) const { + const auto LHSSize = Lines.size(); + const auto RHSSize = RHS.Lines.size(); + if (LHSSize == RHSSize) + return Lines < RHS.Lines; + return LHSSize < RHSSize; + } + Collection::const_iterator begin() const { return Lines.begin(); } + Collection::const_iterator end() const { return Lines.end(); } + +}; + +raw_ostream &operator<<(raw_ostream &OS, const gsym::LineTable <); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_LINETABLE_H diff --git a/include/llvm/DebugInfo/GSYM/Range.h b/include/llvm/DebugInfo/GSYM/Range.h index 772ff244c5b7..37cfec713f26 100644 --- a/include/llvm/DebugInfo/GSYM/Range.h +++ b/include/llvm/DebugInfo/GSYM/Range.h @@ -1,9 +1,8 @@ -//===- AddressRange.h -------------------------------------------*- C++ -*-===// +//===- Range.h --------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,10 +20,13 @@ #define HEX64(v) llvm::format_hex(v, 18) namespace llvm { +class DataExtractor; class raw_ostream; namespace gsym { +class FileWriter; + /// A class that represents an address range. The range is specified using /// a start and an end address. struct AddressRange { @@ -47,6 +49,18 @@ struct AddressRange { bool operator<(const AddressRange &R) const { return std::make_pair(Start, End) < std::make_pair(R.Start, R.End); } + /// AddressRange objects are encoded and decoded to be relative to a base + /// address. This will be the FunctionInfo's start address if the AddressRange + /// is directly contained in a FunctionInfo, or a base address of the + /// containing parent AddressRange or AddressRanges. This allows address + /// ranges to be efficiently encoded using ULEB128 encodings as we encode the + /// offset and size of each range instead of full addresses. This also makes + /// encoded addresses easy to relocate as we just need to relocate one base + /// address. + /// @{ + void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); + void encode(FileWriter &O, uint64_t BaseAddr) const; + /// @} }; raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R); @@ -66,6 +80,7 @@ public: void clear() { Ranges.clear(); } bool empty() const { return Ranges.empty(); } bool contains(uint64_t Addr) const; + bool contains(AddressRange Range) const; void insert(AddressRange Range); size_t size() const { return Ranges.size(); } bool operator==(const AddressRanges &RHS) const { @@ -77,6 +92,14 @@ public: } Collection::const_iterator begin() const { return Ranges.begin(); } Collection::const_iterator end() const { return Ranges.end(); } + + /// Address ranges are decoded and encoded to be relative to a base address. + /// See the AddressRange comment for the encode and decode methods for full + /// details. + /// @{ + void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); + void encode(FileWriter &O, uint64_t BaseAddr) const; + /// @} }; raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR); diff --git a/include/llvm/DebugInfo/GSYM/StringTable.h b/include/llvm/DebugInfo/GSYM/StringTable.h index 0001b8b82743..a96ae5899da3 100644 --- a/include/llvm/DebugInfo/GSYM/StringTable.h +++ b/include/llvm/DebugInfo/GSYM/StringTable.h @@ -1,9 +1,8 @@ //===- StringTable.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/GenericError.h b/include/llvm/DebugInfo/PDB/GenericError.h index ec85d92d2a92..af93be931b8e 100644 --- a/include/llvm/DebugInfo/PDB/GenericError.h +++ b/include/llvm/DebugInfo/PDB/GenericError.h @@ -20,7 +20,7 @@ enum class pdb_error_code { dia_sdk_not_present, dia_failed_loading, signature_out_of_date, - external_cmdline_ref, + no_matching_pch, unspecified, }; } // namespace pdb diff --git a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h index 0b15ab474f71..4adf3b394c2e 100644 --- a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h +++ b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h @@ -87,7 +87,7 @@ public: // Initial construction must not access the cache, since it must be done // atomically. - auto Result = llvm::make_unique<ConcreteSymbolT>( + auto Result = std::make_unique<ConcreteSymbolT>( Session, Id, std::forward<Args>(ConstructorArgs)...); Result->SymbolId = Id; diff --git a/include/llvm/DebugInfo/PDB/PDBSymbol.h b/include/llvm/DebugInfo/PDB/PDBSymbol.h index d9004a8894d9..0d95a2467556 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbol.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbol.h @@ -131,7 +131,7 @@ public: auto BaseIter = RawSymbol->findChildren(T::Tag); if (!BaseIter) return nullptr; - return llvm::make_unique<ConcreteSymbolEnumerator<T>>(std::move(BaseIter)); + return std::make_unique<ConcreteSymbolEnumerator<T>>(std::move(BaseIter)); } std::unique_ptr<IPDBEnumSymbols> findAllChildren(PDB_SymType Type) const; std::unique_ptr<IPDBEnumSymbols> findAllChildren() const; diff --git a/include/llvm/DebugInfo/Symbolize/Symbolize.h b/include/llvm/DebugInfo/Symbolize/Symbolize.h index d3da28ca0b7b..11599fc1797d 100644 --- a/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -39,6 +39,7 @@ public: bool UseSymbolTable = true; bool Demangle = true; bool RelativeAddresses = false; + bool UntagAddresses = false; std::string DefaultArch; std::vector<std::string> DsymHints; std::string FallbackDebugPath; diff --git a/include/llvm/Demangle/Demangle.h b/include/llvm/Demangle/Demangle.h index 6fea7ef13f11..7b85b9a9ccf7 100644 --- a/include/llvm/Demangle/Demangle.h +++ b/include/llvm/Demangle/Demangle.h @@ -32,7 +32,14 @@ char *itaniumDemangle(const char *mangled_name, char *buf, size_t *n, int *status); -enum MSDemangleFlags { MSDF_None = 0, MSDF_DumpBackrefs = 1 << 0 }; +enum MSDemangleFlags { + MSDF_None = 0, + MSDF_DumpBackrefs = 1 << 0, + MSDF_NoAccessSpecifier = 1 << 1, + MSDF_NoCallingConvention = 1 << 2, + MSDF_NoReturnType = 1 << 3, + MSDF_NoMemberType = 1 << 4, +}; char *microsoftDemangle(const char *mangled_name, char *buf, size_t *n, int *status, MSDemangleFlags Flags = MSDF_None); diff --git a/include/llvm/Demangle/DemangleConfig.h b/include/llvm/Demangle/DemangleConfig.h index 73f89d357c85..b7b7dbd24c7f 100644 --- a/include/llvm/Demangle/DemangleConfig.h +++ b/include/llvm/Demangle/DemangleConfig.h @@ -15,13 +15,6 @@ #ifndef LLVM_DEMANGLE_COMPILER_H #define LLVM_DEMANGLE_COMPILER_H -#ifdef _MSC_VER -// snprintf is implemented in VS 2015 -#if _MSC_VER < 1900 -#define snprintf _snprintf_s -#endif -#endif - #ifndef __has_feature #define __has_feature(x) 0 #endif diff --git a/include/llvm/Demangle/ItaniumDemangle.h b/include/llvm/Demangle/ItaniumDemangle.h index aaccb27e17a3..7784e842bfeb 100644 --- a/include/llvm/Demangle/ItaniumDemangle.h +++ b/include/llvm/Demangle/ItaniumDemangle.h @@ -57,6 +57,11 @@ X(LocalName) \ X(VectorType) \ X(PixelVectorType) \ + X(SyntheticTemplateParamName) \ + X(TypeTemplateParamDecl) \ + X(NonTypeTemplateParamDecl) \ + X(TemplateTemplateParamDecl) \ + X(TemplateParamPackDecl) \ X(ParameterPack) \ X(TemplateArgumentPack) \ X(ParameterPackExpansion) \ @@ -91,6 +96,8 @@ X(ThrowExpr) \ X(UUIDOfExpr) \ X(BoolExpr) \ + X(StringLiteral) \ + X(LambdaExpr) \ X(IntegerCastExpr) \ X(IntegerLiteral) \ X(FloatLiteral) \ @@ -303,7 +310,7 @@ inline Qualifiers operator|=(Qualifiers &Q1, Qualifiers Q2) { return Q1 = static_cast<Qualifiers>(Q1 | Q2); } -class QualType : public Node { +class QualType final : public Node { protected: const Qualifiers Quals; const Node *Child; @@ -964,6 +971,127 @@ public: } }; +enum class TemplateParamKind { Type, NonType, Template }; + +/// An invented name for a template parameter for which we don't have a +/// corresponding template argument. +/// +/// This node is created when parsing the <lambda-sig> for a lambda with +/// explicit template arguments, which might be referenced in the parameter +/// types appearing later in the <lambda-sig>. +class SyntheticTemplateParamName final : public Node { + TemplateParamKind Kind; + unsigned Index; + +public: + SyntheticTemplateParamName(TemplateParamKind Kind_, unsigned Index_) + : Node(KSyntheticTemplateParamName), Kind(Kind_), Index(Index_) {} + + template<typename Fn> void match(Fn F) const { F(Kind, Index); } + + void printLeft(OutputStream &S) const override { + switch (Kind) { + case TemplateParamKind::Type: + S += "$T"; + break; + case TemplateParamKind::NonType: + S += "$N"; + break; + case TemplateParamKind::Template: + S += "$TT"; + break; + } + if (Index > 0) + S << Index - 1; + } +}; + +/// A template type parameter declaration, 'typename T'. +class TypeTemplateParamDecl final : public Node { + Node *Name; + +public: + TypeTemplateParamDecl(Node *Name_) + : Node(KTypeTemplateParamDecl, Cache::Yes), Name(Name_) {} + + template<typename Fn> void match(Fn F) const { F(Name); } + + void printLeft(OutputStream &S) const override { + S += "typename "; + } + + void printRight(OutputStream &S) const override { + Name->print(S); + } +}; + +/// A non-type template parameter declaration, 'int N'. +class NonTypeTemplateParamDecl final : public Node { + Node *Name; + Node *Type; + +public: + NonTypeTemplateParamDecl(Node *Name_, Node *Type_) + : Node(KNonTypeTemplateParamDecl, Cache::Yes), Name(Name_), Type(Type_) {} + + template<typename Fn> void match(Fn F) const { F(Name, Type); } + + void printLeft(OutputStream &S) const override { + Type->printLeft(S); + if (!Type->hasRHSComponent(S)) + S += " "; + } + + void printRight(OutputStream &S) const override { + Name->print(S); + Type->printRight(S); + } +}; + +/// A template template parameter declaration, +/// 'template<typename T> typename N'. +class TemplateTemplateParamDecl final : public Node { + Node *Name; + NodeArray Params; + +public: + TemplateTemplateParamDecl(Node *Name_, NodeArray Params_) + : Node(KTemplateTemplateParamDecl, Cache::Yes), Name(Name_), + Params(Params_) {} + + template<typename Fn> void match(Fn F) const { F(Name, Params); } + + void printLeft(OutputStream &S) const override { + S += "template<"; + Params.printWithComma(S); + S += "> typename "; + } + + void printRight(OutputStream &S) const override { + Name->print(S); + } +}; + +/// A template parameter pack declaration, 'typename ...T'. +class TemplateParamPackDecl final : public Node { + Node *Param; + +public: + TemplateParamPackDecl(Node *Param_) + : Node(KTemplateParamPackDecl, Cache::Yes), Param(Param_) {} + + template<typename Fn> void match(Fn F) const { F(Param); } + + void printLeft(OutputStream &S) const override { + Param->printLeft(S); + S += "..."; + } + + void printRight(OutputStream &S) const override { + Param->printRight(S); + } +}; + /// An unexpanded parameter pack (either in the expression or type context). If /// this AST is correct, this node will have a ParameterPackExpansion node above /// it. @@ -1410,21 +1538,36 @@ public: }; class ClosureTypeName : public Node { + NodeArray TemplateParams; NodeArray Params; StringView Count; public: - ClosureTypeName(NodeArray Params_, StringView Count_) - : Node(KClosureTypeName), Params(Params_), Count(Count_) {} + ClosureTypeName(NodeArray TemplateParams_, NodeArray Params_, + StringView Count_) + : Node(KClosureTypeName), TemplateParams(TemplateParams_), + Params(Params_), Count(Count_) {} + + template<typename Fn> void match(Fn F) const { + F(TemplateParams, Params, Count); + } - template<typename Fn> void match(Fn F) const { F(Params, Count); } + void printDeclarator(OutputStream &S) const { + if (!TemplateParams.empty()) { + S += "<"; + TemplateParams.printWithComma(S); + S += ">"; + } + S += "("; + Params.printWithComma(S); + S += ")"; + } void printLeft(OutputStream &S) const override { S += "\'lambda"; S += Count; - S += "\'("; - Params.printWithComma(S); - S += ")"; + S += "\'"; + printDeclarator(S); } }; @@ -1902,6 +2045,37 @@ public: } }; +class StringLiteral : public Node { + const Node *Type; + +public: + StringLiteral(const Node *Type_) : Node(KStringLiteral), Type(Type_) {} + + template<typename Fn> void match(Fn F) const { F(Type); } + + void printLeft(OutputStream &S) const override { + S += "\"<"; + Type->print(S); + S += ">\""; + } +}; + +class LambdaExpr : public Node { + const Node *Type; + +public: + LambdaExpr(const Node *Type_) : Node(KLambdaExpr), Type(Type_) {} + + template<typename Fn> void match(Fn F) const { F(Type); } + + void printLeft(OutputStream &S) const override { + S += "[]"; + if (Type->getKind() == KClosureTypeName) + static_cast<const ClosureTypeName *>(Type)->printDeclarator(S); + S += "{...}"; + } +}; + class IntegerCastExpr : public Node { // ty(integer) const Node *Ty; @@ -2167,10 +2341,36 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { // table. PODSmallVector<Node *, 32> Subs; + using TemplateParamList = PODSmallVector<Node *, 8>; + + class ScopedTemplateParamList { + AbstractManglingParser *Parser; + size_t OldNumTemplateParamLists; + TemplateParamList Params; + + public: + ScopedTemplateParamList(AbstractManglingParser *Parser) + : Parser(Parser), + OldNumTemplateParamLists(Parser->TemplateParams.size()) { + Parser->TemplateParams.push_back(&Params); + } + ~ScopedTemplateParamList() { + assert(Parser->TemplateParams.size() >= OldNumTemplateParamLists); + Parser->TemplateParams.dropBack(OldNumTemplateParamLists); + } + }; + // Template parameter table. Like the above, but referenced like "T42_". // This has a smaller size compared to Subs and Names because it can be // stored on the stack. - PODSmallVector<Node *, 8> TemplateParams; + TemplateParamList OuterTemplateParams; + + // Lists of template parameters indexed by template parameter depth, + // referenced like "TL2_4_". If nonempty, element 0 is always + // OuterTemplateParams; inner elements are always template parameter lists of + // lambda expressions. For a generic lambda with no explicit template + // parameter list, the corresponding parameter list pointer will be null. + PODSmallVector<TemplateParamList *, 4> TemplateParams; // Set of unresolved forward <template-param> references. These can occur in a // conversion operator's type, and are resolved in the enclosing <encoding>. @@ -2178,7 +2378,9 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { bool TryToParseTemplateArgs = true; bool PermitForwardTemplateReferences = false; - bool ParsingLambdaParams = false; + size_t ParsingLambdaParamsAtLevel = (size_t)-1; + + unsigned NumSyntheticTemplateParameters[3] = {}; Alloc ASTAllocator; @@ -2193,9 +2395,11 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Names.clear(); Subs.clear(); TemplateParams.clear(); - ParsingLambdaParams = false; + ParsingLambdaParamsAtLevel = (size_t)-1; TryToParseTemplateArgs = true; PermitForwardTemplateReferences = false; + for (int I = 0; I != 3; ++I) + NumSyntheticTemplateParameters[I] = 0; ASTAllocator.reset(); } @@ -2253,6 +2457,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { bool parseSeqId(size_t *Out); Node *parseSubstitution(); Node *parseTemplateParam(); + Node *parseTemplateParamDecl(); Node *parseTemplateArgs(bool TagTemplates = false); Node *parseTemplateArg(); @@ -2301,9 +2506,10 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { size_t E = ForwardTemplateRefs.size(); for (; I < E; ++I) { size_t Idx = ForwardTemplateRefs[I]->Index; - if (Idx >= TemplateParams.size()) + if (TemplateParams.empty() || !TemplateParams[0] || + Idx >= TemplateParams[0]->size()) return true; - ForwardTemplateRefs[I]->Ref = TemplateParams[Idx]; + ForwardTemplateRefs[I]->Ref = (*TemplateParams[0])[Idx]; } ForwardTemplateRefs.dropBack(State.ForwardTemplateRefsBegin); return false; @@ -2470,7 +2676,12 @@ AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName(NameState *State) { // <lambda-sig> ::= <parameter type>+ # Parameter types or "v" if the lambda has no parameters template <typename Derived, typename Alloc> Node * -AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *) { +AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *State) { + // <template-params> refer to the innermost <template-args>. Clear out any + // outer args that we may have inserted into TemplateParams. + if (State != nullptr) + TemplateParams.clear(); + if (consumeIf("Ut")) { StringView Count = parseNumber(); if (!consumeIf('_')) @@ -2478,22 +2689,59 @@ AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *) { return make<UnnamedTypeName>(Count); } if (consumeIf("Ul")) { - NodeArray Params; - SwapAndRestore<bool> SwapParams(ParsingLambdaParams, true); + SwapAndRestore<size_t> SwapParams(ParsingLambdaParamsAtLevel, + TemplateParams.size()); + ScopedTemplateParamList LambdaTemplateParams(this); + + size_t ParamsBegin = Names.size(); + while (look() == 'T' && + StringView("yptn").find(look(1)) != StringView::npos) { + Node *T = parseTemplateParamDecl(); + if (!T) + return nullptr; + Names.push_back(T); + } + NodeArray TempParams = popTrailingNodeArray(ParamsBegin); + + // FIXME: If TempParams is empty and none of the function parameters + // includes 'auto', we should remove LambdaTemplateParams from the + // TemplateParams list. Unfortunately, we don't find out whether there are + // any 'auto' parameters until too late in an example such as: + // + // template<typename T> void f( + // decltype([](decltype([]<typename T>(T v) {}), + // auto) {})) {} + // template<typename T> void f( + // decltype([](decltype([]<typename T>(T w) {}), + // int) {})) {} + // + // Here, the type of v is at level 2 but the type of w is at level 1. We + // don't find this out until we encounter the type of the next parameter. + // + // However, compilers can't actually cope with the former example in + // practice, and it's likely to be made ill-formed in future, so we don't + // need to support it here. + // + // If we encounter an 'auto' in the function parameter types, we will + // recreate a template parameter scope for it, but any intervening lambdas + // will be parsed in the 'wrong' template parameter depth. + if (TempParams.empty()) + TemplateParams.pop_back(); + if (!consumeIf("vE")) { - size_t ParamsBegin = Names.size(); do { Node *P = getDerived().parseType(); if (P == nullptr) return nullptr; Names.push_back(P); } while (!consumeIf('E')); - Params = popTrailingNodeArray(ParamsBegin); } + NodeArray Params = popTrailingNodeArray(ParamsBegin); + StringView Count = parseNumber(); if (!consumeIf('_')) return nullptr; - return make<ClosureTypeName>(Params, Count); + return make<ClosureTypeName>(TempParams, Params, Count); } if (consumeIf("Ub")) { (void)parseNumber(); @@ -3949,6 +4197,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseConversionExpr() { // ::= L <type> <value float> E # floating literal // ::= L <string type> E # string literal // ::= L <nullptr type> E # nullptr literal (i.e., "LDnE") +// ::= L <lambda type> E # lambda expression // FIXME: ::= L <type> <real-part float> _ <imag-part float> E # complex floating point literal (C 2000) // ::= L <mangled-name> E # external name template <typename Derived, typename Alloc> @@ -4020,24 +4269,43 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExprPrimary() { return R; } return nullptr; + case 'A': { + Node *T = getDerived().parseType(); + if (T == nullptr) + return nullptr; + // FIXME: We need to include the string contents in the mangling. + if (consumeIf('E')) + return make<StringLiteral>(T); + return nullptr; + } + case 'D': + if (consumeIf("DnE")) + return make<NameType>("nullptr"); + return nullptr; case 'T': // Invalid mangled name per // http://sourcerytools.com/pipermail/cxx-abi-dev/2011-August/002422.html return nullptr; + case 'U': { + // FIXME: Should we support LUb... for block literals? + if (look(1) != 'l') + return nullptr; + Node *T = parseUnnamedTypeName(nullptr); + if (!T || !consumeIf('E')) + return nullptr; + return make<LambdaExpr>(T); + } default: { // might be named type Node *T = getDerived().parseType(); if (T == nullptr) return nullptr; StringView N = parseNumber(); - if (!N.empty()) { - if (!consumeIf('E')) - return nullptr; - return make<IntegerCastExpr>(T, N); - } - if (consumeIf('E')) - return T; - return nullptr; + if (N.empty()) + return nullptr; + if (!consumeIf('E')) + return nullptr; + return make<IntegerCastExpr>(T, N); } } } @@ -5062,11 +5330,22 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSubstitution() { // <template-param> ::= T_ # first template parameter // ::= T <parameter-2 non-negative number> _ +// ::= TL <level-1> __ +// ::= TL <level-1> _ <parameter-2 non-negative number> _ template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { if (!consumeIf('T')) return nullptr; + size_t Level = 0; + if (consumeIf('L')) { + if (parsePositiveInteger(&Level)) + return nullptr; + ++Level; + if (!consumeIf('_')) + return nullptr; + } + size_t Index = 0; if (!consumeIf('_')) { if (parsePositiveInteger(&Index)) @@ -5076,15 +5355,11 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { return nullptr; } - // Itanium ABI 5.1.8: In a generic lambda, uses of auto in the parameter list - // are mangled as the corresponding artificial template type parameter. - if (ParsingLambdaParams) - return make<NameType>("auto"); - // If we're in a context where this <template-param> refers to a // <template-arg> further ahead in the mangled name (currently just conversion // operator types), then we should only look it up in the right context. - if (PermitForwardTemplateReferences) { + // This can only happen at the outermost level. + if (PermitForwardTemplateReferences && Level == 0) { Node *ForwardRef = make<ForwardTemplateReference>(Index); if (!ForwardRef) return nullptr; @@ -5094,9 +5369,78 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { return ForwardRef; } - if (Index >= TemplateParams.size()) + if (Level >= TemplateParams.size() || !TemplateParams[Level] || + Index >= TemplateParams[Level]->size()) { + // Itanium ABI 5.1.8: In a generic lambda, uses of auto in the parameter + // list are mangled as the corresponding artificial template type parameter. + if (ParsingLambdaParamsAtLevel == Level && Level <= TemplateParams.size()) { + // This will be popped by the ScopedTemplateParamList in + // parseUnnamedTypeName. + if (Level == TemplateParams.size()) + TemplateParams.push_back(nullptr); + return make<NameType>("auto"); + } + return nullptr; - return TemplateParams[Index]; + } + + return (*TemplateParams[Level])[Index]; +} + +// <template-param-decl> ::= Ty # type parameter +// ::= Tn <type> # non-type parameter +// ::= Tt <template-param-decl>* E # template parameter +// ::= Tp <template-param-decl> # parameter pack +template <typename Derived, typename Alloc> +Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl() { + auto InventTemplateParamName = [&](TemplateParamKind Kind) { + unsigned Index = NumSyntheticTemplateParameters[(int)Kind]++; + Node *N = make<SyntheticTemplateParamName>(Kind, Index); + if (N) TemplateParams.back()->push_back(N); + return N; + }; + + if (consumeIf("Ty")) { + Node *Name = InventTemplateParamName(TemplateParamKind::Type); + if (!Name) + return nullptr; + return make<TypeTemplateParamDecl>(Name); + } + + if (consumeIf("Tn")) { + Node *Name = InventTemplateParamName(TemplateParamKind::NonType); + if (!Name) + return nullptr; + Node *Type = parseType(); + if (!Type) + return nullptr; + return make<NonTypeTemplateParamDecl>(Name, Type); + } + + if (consumeIf("Tt")) { + Node *Name = InventTemplateParamName(TemplateParamKind::Template); + if (!Name) + return nullptr; + size_t ParamsBegin = Names.size(); + ScopedTemplateParamList TemplateTemplateParamParams(this); + while (!consumeIf("E")) { + Node *P = parseTemplateParamDecl(); + if (!P) + return nullptr; + Names.push_back(P); + } + NodeArray Params = popTrailingNodeArray(ParamsBegin); + return make<TemplateTemplateParamDecl>(Name, Params); + } + + if (consumeIf("Tp")) { + Node *P = parseTemplateParamDecl(); + if (!P) + return nullptr; + return make<TemplateParamPackDecl>(P); + } + + return nullptr; } // <template-arg> ::= <type> # type or template @@ -5153,8 +5497,11 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) { // <template-params> refer to the innermost <template-args>. Clear out any // outer args that we may have inserted into TemplateParams. - if (TagTemplates) + if (TagTemplates) { TemplateParams.clear(); + TemplateParams.push_back(&OuterTemplateParams); + OuterTemplateParams.clear(); + } size_t ArgsBegin = Names.size(); while (!consumeIf('E')) { @@ -5172,7 +5519,7 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) { if (!TableEntry) return nullptr; } - TemplateParams.push_back(TableEntry); + TemplateParams.back()->push_back(TableEntry); } else { Node *Arg = getDerived().parseTemplateArg(); if (Arg == nullptr) diff --git a/include/llvm/Demangle/MicrosoftDemangle.h b/include/llvm/Demangle/MicrosoftDemangle.h index 382e79401c43..c6f26061bedd 100644 --- a/include/llvm/Demangle/MicrosoftDemangle.h +++ b/include/llvm/Demangle/MicrosoftDemangle.h @@ -158,6 +158,7 @@ private: QualifiedNameNode *QN); SymbolNode *demangleDeclarator(StringView &MangledName); SymbolNode *demangleMD5Name(StringView &MangledName); + SymbolNode *demangleTypeinfoName(StringView &MangledName); VariableSymbolNode *demangleVariableEncoding(StringView &MangledName, StorageClass SC); diff --git a/include/llvm/Demangle/MicrosoftDemangleNodes.h b/include/llvm/Demangle/MicrosoftDemangleNodes.h index da9d9d5bfdc0..81b279fe237d 100644 --- a/include/llvm/Demangle/MicrosoftDemangleNodes.h +++ b/include/llvm/Demangle/MicrosoftDemangleNodes.h @@ -16,6 +16,8 @@ #include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/StringView.h" #include <array> +#include <cstdint> +#include <string> namespace llvm { namespace itanium_demangle { @@ -73,6 +75,9 @@ enum OutputFlags { OF_Default = 0, OF_NoCallingConvention = 1, OF_NoTagSpecifier = 2, + OF_NoAccessSpecifier = 4, + OF_NoMemberType = 8, + OF_NoReturnType = 16, }; // Types @@ -301,8 +306,6 @@ struct TypeNode : public Node { outputPost(OS, Flags); } - void outputQuals(bool SpaceBefore, bool SpaceAfter) const; - Qualifiers Quals = Q_None; }; diff --git a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h index 8d2f641254b3..72687682f606 100644 --- a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h +++ b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -22,17 +22,21 @@ namespace llvm { namespace jitlink { /// Registers all FDEs in the given eh-frame section with the current process. -Error registerEHFrameSection(const void *EHFrameSectionAddr); +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); /// Deregisters all FDEs in the given eh-frame section with the current process. -Error deregisterEHFrameSection(const void *EHFrameSectionAddr); +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); /// Supports registration/deregistration of EH-frames in a target process. class EHFrameRegistrar { public: virtual ~EHFrameRegistrar(); - virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; - virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; + virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) = 0; + virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) = 0; }; /// Registers / Deregisters EH-frames in the current process. @@ -48,31 +52,38 @@ public: InProcessEHFrameRegistrar(InProcessEHFrameRegistrar &&) = delete; InProcessEHFrameRegistrar &operator=(InProcessEHFrameRegistrar &&) = delete; - Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) override { + Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { return registerEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr)); + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); } - Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) override { + Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { return deregisterEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr)); + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); } private: InProcessEHFrameRegistrar(); }; -using StoreFrameAddressFunction = std::function<void(JITTargetAddress)>; +using StoreFrameRangeFunction = + std::function<void(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize)>; -/// Creates a pass that records the address of the EH frame section. If no -/// eh-frame section is found, it will set EHFrameAddr to zero. +/// Creates a pass that records the address and size of the EH frame section. +/// If no eh-frame section is found then the address and size will both be given +/// as zero. /// /// Authors of JITLinkContexts can use this function to register a post-fixup -/// pass that records the address of the eh-frame section. This address can +/// pass that records the range of the eh-frame section. This range can /// be used after finalization to register and deregister the frame. -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, - StoreFrameAddressFunction StoreFrameAddress); + StoreFrameRangeFunction StoreFrameRange); } // end namespace jitlink } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index be80d44ccf51..b531127cf892 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -34,6 +34,9 @@ namespace llvm { namespace jitlink { +class Symbol; +class Section; + /// Base class for errors originating in JIT linker, e.g. missing relocation /// support. class JITLinkError : public ErrorInfo<JITLinkError> { @@ -50,27 +53,22 @@ private: std::string ErrMsg; }; -// Forward declare the Atom class. -class Atom; - -/// Edge class. Represents both object file relocations, as well as layout and -/// keep-alive constraints. +/// Represents fixups and constraints in the LinkGraph. class Edge { public: using Kind = uint8_t; - using GenericEdgeKind = enum : Kind { + enum GenericEdgeKind : Kind { Invalid, // Invalid edge value. FirstKeepAlive, // Keeps target alive. Offset/addend zero. KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. - LayoutNext, // Layout constraint. Offset/Addend zero. FirstRelocation // First architecture specific relocation. }; using OffsetT = uint32_t; using AddendT = int64_t; - Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + Edge(Kind K, OffsetT Offset, Symbol &Target, AddendT Addend) : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} OffsetT getOffset() const { return Offset; } @@ -82,461 +80,637 @@ public: return K - FirstRelocation; } bool isKeepAlive() const { return K >= FirstKeepAlive; } - Atom &getTarget() const { return *Target; } - void setTarget(Atom &Target) { this->Target = &Target; } + Symbol &getTarget() const { return *Target; } + void setTarget(Symbol &Target) { this->Target = &Target; } AddendT getAddend() const { return Addend; } void setAddend(AddendT Addend) { this->Addend = Addend; } private: - Atom *Target; - OffsetT Offset; - AddendT Addend; + Symbol *Target = nullptr; + OffsetT Offset = 0; + AddendT Addend = 0; Kind K = 0; }; -using EdgeVector = std::vector<Edge>; +/// Returns the string name of the given generic edge kind, or "unknown" +/// otherwise. Useful for debugging. +const char *getGenericEdgeKindName(Edge::Kind K); -const StringRef getGenericEdgeKindName(Edge::Kind K); - -/// Base Atom class. Used by absolute and undefined atoms. -class Atom { - friend class AtomGraph; +/// Base class for Addressable entities (externals, absolutes, blocks). +class Addressable { + friend class LinkGraph; protected: - /// Create a named (as yet unresolved) atom. - Atom(StringRef Name) - : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false), - IsGlobal(false), IsAbsolute(false), IsCallable(false), - IsExported(false), IsWeak(false), HasLayoutNext(false), - IsCommon(false) {} - - /// Create an absolute symbol atom. - Atom(StringRef Name, JITTargetAddress Address) - : Name(Name), Address(Address), IsDefined(true), IsLive(false), - ShouldDiscard(false), IsGlobal(false), IsAbsolute(false), - IsCallable(false), IsExported(false), IsWeak(false), - HasLayoutNext(false), IsCommon(false) {} + Addressable(JITTargetAddress Address, bool IsDefined) + : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {} -public: - /// Returns true if this atom has a name. - bool hasName() const { return Name != StringRef(); } + Addressable(JITTargetAddress Address) + : Address(Address), IsDefined(false), IsAbsolute(true) { + assert(!(IsDefined && IsAbsolute) && + "Block cannot be both defined and absolute"); + } - /// Returns the name of this atom. - StringRef getName() const { return Name; } +public: + Addressable(const Addressable &) = delete; + Addressable &operator=(const Addressable &) = default; + Addressable(Addressable &&) = delete; + Addressable &operator=(Addressable &&) = default; - /// Returns the current target address of this atom. - /// The initial target address (for atoms that have one) will be taken from - /// the input object file's virtual address space. During the layout phase - /// of JIT linking the atom's address will be updated to point to its final - /// address in the JIT'd process. JITTargetAddress getAddress() const { return Address; } - - /// Set the current target address of this atom. void setAddress(JITTargetAddress Address) { this->Address = Address; } - /// Returns true if this is a defined atom. - bool isDefined() const { return IsDefined; } + /// Returns true if this is a defined addressable, in which case you + /// can downcast this to a . + bool isDefined() const { return static_cast<bool>(IsDefined); } + bool isAbsolute() const { return static_cast<bool>(IsAbsolute); } - /// Returns true if this atom is marked as live. - bool isLive() const { return IsLive; } +private: + JITTargetAddress Address = 0; + uint64_t IsDefined : 1; + uint64_t IsAbsolute : 1; +}; - /// Mark this atom as live. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setLive(bool IsLive) { - assert((IsDefined || IsAbsolute || !IsLive) && - "Only defined and absolute atoms can be marked live"); - this->IsLive = IsLive; - } +using BlockOrdinal = unsigned; +using SectionOrdinal = unsigned; - /// Returns true if this atom should be discarded during pruning. - bool shouldDiscard() const { return ShouldDiscard; } +/// An Addressable with content and edges. +class Block : public Addressable { + friend class LinkGraph; - /// Mark this atom to be discarded. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setShouldDiscard(bool ShouldDiscard) { - assert((IsDefined || IsAbsolute || !ShouldDiscard) && - "Only defined and absolute atoms can be marked live"); - this->ShouldDiscard = ShouldDiscard; +private: + /// Create a zero-fill defined addressable. + Block(Section &Parent, BlockOrdinal Ordinal, JITTargetAddress Size, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Size(Size), + Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; } - /// Returns true if this definition is global (i.e. visible outside this - /// linkage unit). - /// - /// Note: This is distict from Exported, which means visibile outside the - /// JITDylib that this graph is being linked in to. - bool isGlobal() const { return IsGlobal; } + /// Create a defined addressable for the given content. + Block(Section &Parent, BlockOrdinal Ordinal, StringRef Content, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Data(Content.data()), + Size(Content.size()), Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } - /// Mark this atom as global. - void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; } +public: + using EdgeVector = std::vector<Edge>; + using edge_iterator = EdgeVector::iterator; + using const_edge_iterator = EdgeVector::const_iterator; - /// Returns true if this atom represents an absolute symbol. - bool isAbsolute() const { return IsAbsolute; } + Block(const Block &) = delete; + Block &operator=(const Block &) = delete; + Block(Block &&) = delete; + Block &operator=(Block &&) = delete; - /// Returns true if this atom is known to be callable. + /// Return the parent section for this block. + Section &getSection() const { return Parent; } + + /// Return the ordinal for this block. + BlockOrdinal getOrdinal() const { return Ordinal; } + + /// Returns true if this is a zero-fill block. /// - /// Primarily provided for easy interoperability with ORC, which uses the - /// JITSymbolFlags::Common flag to identify symbols that can be interposed - /// with stubs. - bool isCallable() const { return IsCallable; } + /// If true, getSize is callable but getContent is not (the content is + /// defined to be a sequence of zero bytes of length Size). + bool isZeroFill() const { return !Data; } + + /// Returns the size of this defined addressable. + size_t getSize() const { return Size; } + + /// Get the content for this block. Block must not be a zero-fill block. + StringRef getContent() const { + assert(Data && "Section does not contain content"); + return StringRef(Data, Size); + } - /// Mark this atom as callable. - void setCallable(bool IsCallable) { - assert((IsDefined || IsAbsolute || !IsCallable) && - "Callable atoms must be defined or absolute"); - this->IsCallable = IsCallable; + /// Set the content for this block. + /// Caller is responsible for ensuring the underlying bytes are not + /// deallocated while pointed to by this block. + void setContent(StringRef Content) { + Data = Content.data(); + Size = Content.size(); } - /// Returns true if this atom should appear in the symbol table of a final - /// linked image. - bool isExported() const { return IsExported; } + /// Get the alignment for this content. + uint64_t getAlignment() const { return 1ull << P2Align; } + + /// Get the alignment offset for this content. + uint64_t getAlignmentOffset() const { return AlignmentOffset; } - /// Mark this atom as exported. - void setExported(bool IsExported) { - assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) && - "Exported atoms must have names"); - this->IsExported = IsExported; + /// Add an edge to this block. + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, + Edge::AddendT Addend) { + Edges.push_back(Edge(K, Offset, Target, Addend)); } - /// Returns true if this is a weak symbol. - bool isWeak() const { return IsWeak; } + /// Return the list of edges attached to this content. + iterator_range<edge_iterator> edges() { + return make_range(Edges.begin(), Edges.end()); + } - /// Mark this atom as weak. - void setWeak(bool IsWeak) { this->IsWeak = IsWeak; } + /// Returns the list of edges attached to this content. + iterator_range<const_edge_iterator> edges() const { + return make_range(Edges.begin(), Edges.end()); + } -private: - StringRef Name; - JITTargetAddress Address = 0; + /// Return the size of the edges list. + size_t edges_size() const { return Edges.size(); } - bool IsDefined : 1; - bool IsLive : 1; - bool ShouldDiscard : 1; + /// Returns true if the list of edges is empty. + bool edges_empty() const { return Edges.empty(); } - bool IsGlobal : 1; - bool IsAbsolute : 1; - bool IsCallable : 1; - bool IsExported : 1; - bool IsWeak : 1; +private: + static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1; -protected: - // These flags only make sense for DefinedAtom, but we can minimize the size - // of DefinedAtom by defining them here. - bool HasLayoutNext : 1; - bool IsCommon : 1; + uint64_t P2Align : 5; + uint64_t AlignmentOffset : 57; + Section &Parent; + const char *Data = nullptr; + size_t Size = 0; + BlockOrdinal Ordinal = 0; + std::vector<Edge> Edges; }; -// Forward declare DefinedAtom. -class DefinedAtom; +/// Describes symbol linkage. This can be used to make resolve definition +/// clashes. +enum class Linkage : uint8_t { + Strong, + Weak, +}; -raw_ostream &operator<<(raw_ostream &OS, const Atom &A); -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, - StringRef EdgeKindName); +/// For errors and debugging output. +const char *getLinkageName(Linkage L); + +/// Defines the scope in which this symbol should be visible: +/// Default -- Visible in the public interface of the linkage unit. +/// Hidden -- Visible within the linkage unit, but not exported from it. +/// Local -- Visible only within the LinkGraph. +enum class Scope : uint8_t { Default, Hidden, Local }; + +/// For debugging output. +const char *getScopeName(Scope S); + +raw_ostream &operator<<(raw_ostream &OS, const Block &B); + +/// Symbol representation. +/// +/// Symbols represent locations within Addressable objects. +/// They can be either Named or Anonymous. +/// Anonymous symbols have neither linkage nor visibility, and must point at +/// ContentBlocks. +/// Named symbols may be in one of four states: +/// - Null: Default initialized. Assignable, but otherwise unusable. +/// - Defined: Has both linkage and visibility and points to a ContentBlock +/// - Common: Has both linkage and visibility, points to a null Addressable. +/// - External: Has neither linkage nor visibility, points to an external +/// Addressable. +/// +class Symbol { + friend class LinkGraph; + +private: + Symbol(Addressable &Base, JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, bool IsLive, + bool IsCallable) + : Name(Name), Base(&Base), Offset(Offset), Size(Size) { + setLinkage(L); + setScope(S); + setLive(IsLive); + setCallable(IsCallable); + } + + static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name, + JITTargetAddress Size, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Common symbol name cannot be empty"); + assert(Base.isDefined() && + "Cannot create common symbol from undefined block"); + assert(static_cast<Block &>(Base).getSize() == Size && + "Common symbol size should match underlying block size"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Weak, S, IsLive, false); + return *Sym; + } + + static Symbol &constructExternal(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create external symbol from defined block"); + assert(!Name.empty() && "External symbol name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Strong, Scope::Default, + false, false); + return *Sym; + } + + static Symbol &constructAbsolute(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size, + Linkage L, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create absolute symbol from a defined block"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false); + return *Sym; + } + + static Symbol &constructAnonDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, + Scope::Local, IsLive, IsCallable); + return *Sym; + } + + static Symbol &constructNamedDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, + bool IsLive, bool IsCallable) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); + return *Sym; + } -/// Represents a section address range via a pair of DefinedAtom pointers to -/// the first and last atoms in the section. -class SectionRange { public: - SectionRange() = default; - SectionRange(DefinedAtom *First, DefinedAtom *Last) - : First(First), Last(Last) {} - DefinedAtom *getFirstAtom() const { - assert((!Last || First) && "First can not be null if end is non-null"); - return First; + /// Create a null Symbol. This allows Symbols to be default initialized for + /// use in containers (e.g. as map values). Null symbols are only useful for + /// assigning to. + Symbol() = default; + + // Symbols are not movable or copyable. + Symbol(const Symbol &) = delete; + Symbol &operator=(const Symbol &) = delete; + Symbol(Symbol &&) = delete; + Symbol &operator=(Symbol &&) = delete; + + /// Returns true if this symbol has a name. + bool hasName() const { return !Name.empty(); } + + /// Returns the name of this symbol (empty if the symbol is anonymous). + StringRef getName() const { + assert((!Name.empty() || getScope() == Scope::Local) && + "Anonymous symbol has non-local scope"); + return Name; } - DefinedAtom *getLastAtom() const { - assert((First || !Last) && "Last can not be null if start is non-null"); - return Last; + + /// Returns true if this Symbol has content (potentially) defined within this + /// object file (i.e. is anything but an external or absolute symbol). + bool isDefined() const { + assert(Base && "Attempt to access null symbol"); + return Base->isDefined(); } - bool isEmpty() const { - assert((First || !Last) && "Last can not be null if start is non-null"); - return !First; + + /// Returns true if this symbol is live (i.e. should be treated as a root for + /// dead stripping). + bool isLive() const { + assert(Base && "Attempting to access null symbol"); + return IsLive; } - JITTargetAddress getStart() const; - JITTargetAddress getEnd() const; - uint64_t getSize() const; -private: - DefinedAtom *First = nullptr; - DefinedAtom *Last = nullptr; -}; + /// Set this symbol's live bit. + void setLive(bool IsLive) { this->IsLive = IsLive; } -/// Represents an object file section. -class Section { - friend class AtomGraph; + /// Returns true is this symbol is callable. + bool isCallable() const { return IsCallable; } -private: - Section(StringRef Name, uint32_t Alignment, sys::Memory::ProtectionFlags Prot, - unsigned Ordinal, bool IsZeroFill) - : Name(Name), Alignment(Alignment), Prot(Prot), Ordinal(Ordinal), - IsZeroFill(IsZeroFill) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of 2"); + /// Set this symbol's callable bit. + void setCallable(bool IsCallable) { this->IsCallable = IsCallable; } + + /// Returns true if the underlying addressable is an unresolved external. + bool isExternal() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && !Base->isAbsolute(); } - using DefinedAtomSet = DenseSet<DefinedAtom *>; + /// Returns true if the underlying addressable is an absolute symbol. + bool isAbsolute() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && Base->isAbsolute(); + } -public: - using atom_iterator = DefinedAtomSet::iterator; - using const_atom_iterator = DefinedAtomSet::const_iterator; + /// Return the addressable that this symbol points to. + Addressable &getAddressable() { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } - ~Section(); - StringRef getName() const { return Name; } - uint32_t getAlignment() const { return Alignment; } - sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } - unsigned getSectionOrdinal() const { return Ordinal; } - size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } + /// Return the addressable that thsi symbol points to. + const Addressable &getAddressable() const { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } - bool isZeroFill() const { return IsZeroFill; } + /// Return the Block for this Symbol (Symbol must be defined). + Block &getBlock() { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<Block &>(*Base); + } - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range<atom_iterator> atoms() { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + /// Return the Block for this Symbol (Symbol must be defined). + const Block &getBlock() const { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<const Block &>(*Base); } - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range<const_atom_iterator> atoms() const { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + /// Returns the offset for this symbol within the underlying addressable. + JITTargetAddress getOffset() const { return Offset; } + + /// Returns the address of this symbol. + JITTargetAddress getAddress() const { return Base->getAddress() + Offset; } + + /// Returns the size of this symbol. + JITTargetAddress getSize() const { return Size; } + + /// Returns true if this symbol is backed by a zero-fill block. + /// This method may only be called on defined symbols. + bool isSymbolZeroFill() const { return getBlock().isZeroFill(); } + + /// Returns the content in the underlying block covered by this symbol. + /// This method may only be called on defined non-zero-fill symbols. + StringRef getSymbolContent() const { + return getBlock().getContent().substr(Offset, Size); } - /// Return the number of atoms in this section. - DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); } + /// Get the linkage for this Symbol. + Linkage getLinkage() const { return static_cast<Linkage>(L); } - /// Return true if this section contains no atoms. - bool atoms_empty() const { return DefinedAtoms.empty(); } + /// Set the linkage for this Symbol. + void setLinkage(Linkage L) { + assert((L == Linkage::Strong || (Base->isDefined() && !Name.empty())) && + "Linkage can only be applied to defined named symbols"); + this->L = static_cast<uint8_t>(L); + } - /// Returns the range of this section as the pair of atoms with the lowest - /// and highest target address. This operation is expensive, as it - /// must traverse all atoms in the section. - /// - /// Note: If the section is empty, both values will be null. The section - /// address will evaluate to null, and the size to zero. If the section - /// contains a single atom both values will point to it, the address will - /// evaluate to the address of that atom, and the size will be the size of - /// that atom. - SectionRange getRange() const; + /// Get the visibility for this Symbol. + Scope getScope() const { return static_cast<Scope>(S); } -private: - void addAtom(DefinedAtom &DA) { - assert(!DefinedAtoms.count(&DA) && "Atom is already in this section"); - DefinedAtoms.insert(&DA); + /// Set the visibility for this Symbol. + void setScope(Scope S) { + assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && + "Invalid visibility for symbol type"); + this->S = static_cast<uint8_t>(S); } - void removeAtom(DefinedAtom &DA) { - assert(DefinedAtoms.count(&DA) && "Atom is not in this section"); - DefinedAtoms.erase(&DA); +private: + void makeExternal(Addressable &A) { + assert(!A.isDefined() && "Attempting to make external with defined block"); + Base = &A; + Offset = 0; + setLinkage(Linkage::Strong); + setScope(Scope::Default); + IsLive = 0; + // note: Size and IsCallable fields left unchanged. } + static constexpr uint64_t MaxOffset = (1ULL << 59) - 1; + + // FIXME: A char* or SymbolStringPtr may pack better. StringRef Name; - uint32_t Alignment = 0; - sys::Memory::ProtectionFlags Prot; - unsigned Ordinal = 0; - unsigned NextAtomOrdinal = 0; - bool IsZeroFill = false; - DefinedAtomSet DefinedAtoms; + Addressable *Base = nullptr; + uint64_t Offset : 59; + uint64_t L : 1; + uint64_t S : 2; + uint64_t IsLive : 1; + uint64_t IsCallable : 1; + JITTargetAddress Size = 0; }; -/// Defined atom class. Suitable for use by defined named and anonymous -/// atoms. -class DefinedAtom : public Atom { - friend class AtomGraph; +raw_ostream &operator<<(raw_ostream &OS, const Symbol &A); + +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, + StringRef EdgeKindName); + +/// Represents an object file section. +class Section { + friend class LinkGraph; private: - DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) - : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), - Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } + Section(StringRef Name, sys::Memory::ProtectionFlags Prot, + SectionOrdinal SecOrdinal) + : Name(Name), Prot(Prot), SecOrdinal(SecOrdinal) {} - DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, - uint32_t Alignment) - : Atom(Name, Address), Parent(Parent), - Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } + using SymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; public: - using edge_iterator = EdgeVector::iterator; + using symbol_iterator = SymbolSet::iterator; + using const_symbol_iterator = SymbolSet::const_iterator; - Section &getSection() const { return Parent; } + using block_iterator = BlockSet::iterator; + using const_block_iterator = BlockSet::const_iterator; - uint64_t getSize() const { return Size; } + ~Section(); - StringRef getContent() const { - assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom"); - assert(Size <= std::numeric_limits<size_t>::max() && - "Content size too large"); - return {ContentPtr, static_cast<size_t>(Size)}; - } - void setContent(StringRef Content) { - assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?"); - ContentPtr = Content.data(); - Size = Content.size(); - } + /// Returns the name of this section. + StringRef getName() const { return Name; } + + /// Returns the protection flags for this section. + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } - bool isZeroFill() const { return Parent.isZeroFill(); } + /// Returns the ordinal for this section. + SectionOrdinal getOrdinal() const { return SecOrdinal; } - void setZeroFill(uint64_t Size) { - assert(Parent.isZeroFill() && !ContentPtr && - "Can't set zero-fill length of a non zero-fill atom"); - this->Size = Size; + /// Returns an iterator over the symbols defined in this section. + iterator_range<symbol_iterator> symbols() { + return make_range(Symbols.begin(), Symbols.end()); } - uint64_t getZeroFillSize() const { - assert(Parent.isZeroFill() && - "Can't get zero-fill length of a non zero-fill atom"); - return Size; + /// Returns an iterator over the symbols defined in this section. + iterator_range<const_symbol_iterator> symbols() const { + return make_range(Symbols.begin(), Symbols.end()); } - uint32_t getAlignment() const { return Alignment; } + /// Return the number of symbols in this section. + SymbolSet::size_type symbols_size() { return Symbols.size(); } - bool hasLayoutNext() const { return HasLayoutNext; } - void setLayoutNext(DefinedAtom &Next) { - assert(!HasLayoutNext && "Atom already has layout-next constraint"); - HasLayoutNext = true; - Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0)); - } - DefinedAtom &getLayoutNext() { - assert(HasLayoutNext && "Atom does not have a layout-next constraint"); - DefinedAtom *Next = nullptr; - for (auto &E : edges()) - if (E.getKind() == Edge::LayoutNext) { - assert(E.getTarget().isDefined() && - "layout-next target atom must be a defined atom"); - Next = static_cast<DefinedAtom *>(&E.getTarget()); - break; - } - assert(Next && "Missing LayoutNext edge"); - return *Next; - } + /// Return true if this section contains no symbols. + bool symbols_empty() const { return Symbols.empty(); } - bool isCommon() const { return IsCommon; } + /// Returns the ordinal for the next block. + BlockOrdinal getNextBlockOrdinal() { return NextBlockOrdinal++; } - void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target, - Edge::AddendT Addend) { - assert(K != Edge::LayoutNext && - "Layout edges should be added via setLayoutNext"); - Edges.push_back(Edge(K, Offset, Target, Addend)); +private: + void addSymbol(Symbol &Sym) { + assert(!Symbols.count(&Sym) && "Symbol is already in this section"); + Symbols.insert(&Sym); } - iterator_range<edge_iterator> edges() { - return make_range(Edges.begin(), Edges.end()); + void removeSymbol(Symbol &Sym) { + assert(Symbols.count(&Sym) && "symbol is not in this section"); + Symbols.erase(&Sym); } - size_t edges_size() const { return Edges.size(); } - bool edges_empty() const { return Edges.empty(); } - unsigned getOrdinal() const { return Ordinal; } + StringRef Name; + sys::Memory::ProtectionFlags Prot; + SectionOrdinal SecOrdinal = 0; + BlockOrdinal NextBlockOrdinal = 0; + SymbolSet Symbols; +}; -private: - void setCommon(uint64_t Size) { - assert(ContentPtr == 0 && "Atom already has content?"); - IsCommon = true; - setZeroFill(Size); +/// Represents a section address range via a pair of Block pointers +/// to the first and last Blocks in the section. +class SectionRange { +public: + SectionRange() = default; + SectionRange(const Section &Sec) { + if (Sec.symbols_empty()) + return; + First = Last = *Sec.symbols().begin(); + for (auto *Sym : Sec.symbols()) { + if (Sym->getAddress() < First->getAddress()) + First = Sym; + if (Sym->getAddress() > Last->getAddress()) + Last = Sym; + } + } + Symbol *getFirstSymbol() const { + assert((!Last || First) && "First can not be null if end is non-null"); + return First; + } + Symbol *getLastSymbol() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return Last; + } + bool isEmpty() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return !First; + } + JITTargetAddress getStart() const { + return First ? First->getBlock().getAddress() : 0; + } + JITTargetAddress getEnd() const { + return Last ? Last->getBlock().getAddress() + Last->getBlock().getSize() + : 0; } + uint64_t getSize() const { return getEnd() - getStart(); } - EdgeVector Edges; - uint64_t Size = 0; - Section &Parent; - const char *ContentPtr = nullptr; - unsigned Ordinal = 0; - uint32_t Alignment = 0; +private: + Symbol *First = nullptr; + Symbol *Last = nullptr; }; -inline JITTargetAddress SectionRange::getStart() const { - return First ? First->getAddress() : 0; -} +class LinkGraph { +private: + using SectionList = std::vector<std::unique_ptr<Section>>; + using ExternalSymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; + + template <typename... ArgTs> + Addressable &createAddressable(ArgTs &&... Args) { + Addressable *A = + reinterpret_cast<Addressable *>(Allocator.Allocate<Addressable>()); + new (A) Addressable(std::forward<ArgTs>(Args)...); + return *A; + } -inline JITTargetAddress SectionRange::getEnd() const { - return Last ? Last->getAddress() + Last->getSize() : 0; -} + void destroyAddressable(Addressable &A) { + A.~Addressable(); + Allocator.Deallocate(&A); + } -inline uint64_t SectionRange::getSize() const { return getEnd() - getStart(); } + template <typename... ArgTs> Block &createBlock(ArgTs &&... Args) { + Block *B = reinterpret_cast<Block *>(Allocator.Allocate<Block>()); + new (B) Block(std::forward<ArgTs>(Args)...); + Blocks.insert(B); + return *B; + } -inline SectionRange Section::getRange() const { - if (atoms_empty()) - return SectionRange(); - DefinedAtom *First = *DefinedAtoms.begin(), *Last = *DefinedAtoms.begin(); - for (auto *DA : atoms()) { - if (DA->getAddress() < First->getAddress()) - First = DA; - if (DA->getAddress() > Last->getAddress()) - Last = DA; + void destroyBlock(Block &B) { + Blocks.erase(&B); + B.~Block(); + Allocator.Deallocate(&B); } - return SectionRange(First, Last); -} -class AtomGraph { -private: - using SectionList = std::vector<std::unique_ptr<Section>>; - using AddressToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - using NamedAtomMap = DenseMap<StringRef, Atom *>; - using ExternalAtomSet = DenseSet<Atom *>; + void destroySymbol(Symbol &S) { + S.~Symbol(); + Allocator.Deallocate(&S); + } public: - using external_atom_iterator = ExternalAtomSet::iterator; + using external_symbol_iterator = ExternalSymbolSet::iterator; + + using block_iterator = BlockSet::iterator; using section_iterator = pointee_iterator<SectionList::iterator>; using const_section_iterator = pointee_iterator<SectionList::const_iterator>; - template <typename SecItrT, typename AtomItrT, typename T> - class defined_atom_iterator_impl + template <typename SectionItrT, typename SymbolItrT, typename T> + class defined_symbol_iterator_impl : public iterator_facade_base< - defined_atom_iterator_impl<SecItrT, AtomItrT, T>, + defined_symbol_iterator_impl<SectionItrT, SymbolItrT, T>, std::forward_iterator_tag, T> { public: - defined_atom_iterator_impl() = default; + defined_symbol_iterator_impl() = default; - defined_atom_iterator_impl(SecItrT SI, SecItrT SE) - : SI(SI), SE(SE), - AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) { - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl(SectionItrT SecI, SectionItrT SecE) + : SecI(SecI), SecE(SecE), + SymI(SecI != SecE ? SecI->symbols().begin() : SymbolItrT()) { + moveToNextSymbolOrEnd(); } - bool operator==(const defined_atom_iterator_impl &RHS) const { - return (SI == RHS.SI) && (AI == RHS.AI); + bool operator==(const defined_symbol_iterator_impl &RHS) const { + return (SecI == RHS.SecI) && (SymI == RHS.SymI); } T operator*() const { - assert(AI != SI->atoms().end() && "Dereferencing end?"); - return *AI; + assert(SymI != SecI->symbols().end() && "Dereferencing end?"); + return *SymI; } - defined_atom_iterator_impl operator++() { - ++AI; - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl operator++() { + ++SymI; + moveToNextSymbolOrEnd(); return *this; } private: - void moveToNextAtomOrEnd() { - while (SI != SE && AI == SI->atoms().end()) { - ++SI; - if (SI == SE) - AI = Section::atom_iterator(); - else - AI = SI->atoms().begin(); + void moveToNextSymbolOrEnd() { + while (SecI != SecE && SymI == SecI->symbols().end()) { + ++SecI; + SymI = SecI == SecE ? SymbolItrT() : SecI->symbols().begin(); } } - SecItrT SI, SE; - AtomItrT AI; + SectionItrT SecI, SecE; + SymbolItrT SymI; }; - using defined_atom_iterator = - defined_atom_iterator_impl<section_iterator, Section::atom_iterator, - DefinedAtom *>; + using defined_symbol_iterator = + defined_symbol_iterator_impl<const_section_iterator, + Section::symbol_iterator, Symbol *>; - using const_defined_atom_iterator = - defined_atom_iterator_impl<const_section_iterator, - Section::const_atom_iterator, - const DefinedAtom *>; + using const_defined_symbol_iterator = defined_symbol_iterator_impl< + const_section_iterator, Section::const_symbol_iterator, const Symbol *>; - AtomGraph(std::string Name, unsigned PointerSize, + LinkGraph(std::string Name, unsigned PointerSize, support::endianness Endianness) : Name(std::move(Name)), PointerSize(PointerSize), Endianness(Endianness) {} + ~LinkGraph(); + /// Returns the name of this graph (usually the name of the original /// underlying MemoryBuffer). const std::string &getName() { return Name; } @@ -544,84 +718,83 @@ public: /// Returns the pointer size for use in this graph. unsigned getPointerSize() const { return PointerSize; } - /// Returns the endianness of atom-content in this graph. + /// Returns the endianness of content in this graph. support::endianness getEndianness() const { return Endianness; } /// Create a section with the given name, protection flags, and alignment. - Section &createSection(StringRef Name, uint32_t Alignment, - sys::Memory::ProtectionFlags Prot, bool IsZeroFill) { - std::unique_ptr<Section> Sec( - new Section(Name, Alignment, Prot, Sections.size(), IsZeroFill)); + Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot) { + std::unique_ptr<Section> Sec(new Section(Name, Prot, Sections.size())); Sections.push_back(std::move(Sec)); return *Sections.back(); } - /// Add an external atom representing an undefined symbol in this graph. - Atom &addExternalAtom(StringRef Name) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast<Atom *>( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name); - ExternalAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a content block. + Block &createContentBlock(Section &Parent, StringRef Content, + uint64_t Address, uint64_t Alignment, + uint64_t AlignmentOffset) { + return createBlock(Parent, Parent.getNextBlockOrdinal(), Content, Address, + Alignment, AlignmentOffset); } - /// Add an external atom representing an absolute symbol. - Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast<Atom *>( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name, Addr); - AbsoluteAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a zero-fill block. + Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address, + uint64_t Alignment, uint64_t AlignmentOffset) { + return createBlock(Parent, Parent.getNextBlockOrdinal(), Size, Address, + Alignment, AlignmentOffset); } - /// Add an anonymous defined atom to the graph. - /// - /// Anonymous atoms have content but no name. They must have an address. - DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, - uint32_t Alignment) { - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - return *A; + /// Add an external symbol. + /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose + /// size is not known, you should substitute '0'. + Symbol &addExternalSymbol(StringRef Name, uint64_t Size) { + auto &Sym = Symbol::constructExternal( + Allocator.Allocate<Symbol>(), createAddressable(0, false), Name, Size); + ExternalSymbols.insert(&Sym); + return Sym; } - /// Add a defined atom to the graph. - /// - /// Allocates and constructs a DefinedAtom instance with the given parent, - /// name, address, and alignment. - DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - NamedAtoms[Name] = A; - return *A; + /// Add an absolute symbol. + Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address, + uint64_t Size, Linkage L, Scope S, bool IsLive) { + auto &Sym = Symbol::constructAbsolute(Allocator.Allocate<Symbol>(), + createAddressable(Address), Name, + Size, L, S, IsLive); + AbsoluteSymbols.insert(&Sym); + return Sym; } - /// Add a common symbol atom to the graph. - /// - /// Adds a common-symbol atom to the graph with the given parent, name, - /// address, alignment and size. - DefinedAtom &addCommonAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment, - uint64_t Size) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - A->setCommon(Size); - Parent.addAtom(*A); - NamedAtoms[Name] = A; - return *A; + /// Convenience method for adding a weak zero-fill symbol. + Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section, + JITTargetAddress Address, uint64_t Size, + uint64_t Alignment, bool IsLive) { + auto &Sym = Symbol::constructCommon( + Allocator.Allocate<Symbol>(), + createBlock(Section, Section.getNextBlockOrdinal(), Address, Size, + Alignment, 0), + Name, Size, S, IsLive); + Section.addSymbol(Sym); + return Sym; + } + + /// Add an anonymous symbol. + Symbol &addAnonymousSymbol(Block &Content, JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + auto &Sym = Symbol::constructAnonDef(Allocator.Allocate<Symbol>(), Content, + Offset, Size, IsCallable, IsLive); + Content.getSection().addSymbol(Sym); + return Sym; + } + + /// Add a named symbol. + Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset, + StringRef Name, JITTargetAddress Size, Linkage L, + Scope S, bool IsCallable, bool IsLive) { + auto &Sym = + Symbol::constructNamedDef(Allocator.Allocate<Symbol>(), Content, Offset, + Name, Size, L, S, IsLive, IsCallable); + Content.getSection().addSymbol(Sym); + return Sym; } iterator_range<section_iterator> sections() { @@ -638,135 +811,79 @@ public: return nullptr; } - iterator_range<external_atom_iterator> external_atoms() { - return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + iterator_range<external_symbol_iterator> external_symbols() { + return make_range(ExternalSymbols.begin(), ExternalSymbols.end()); } - iterator_range<external_atom_iterator> absolute_atoms() { - return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end()); + iterator_range<external_symbol_iterator> absolute_symbols() { + return make_range(AbsoluteSymbols.begin(), AbsoluteSymbols.end()); } - iterator_range<defined_atom_iterator> defined_atoms() { - return make_range(defined_atom_iterator(Sections.begin(), Sections.end()), - defined_atom_iterator(Sections.end(), Sections.end())); + iterator_range<defined_symbol_iterator> defined_symbols() { + return make_range(defined_symbol_iterator(Sections.begin(), Sections.end()), + defined_symbol_iterator(Sections.end(), Sections.end())); } - iterator_range<const_defined_atom_iterator> defined_atoms() const { + iterator_range<const_defined_symbol_iterator> defined_symbols() const { return make_range( - const_defined_atom_iterator(Sections.begin(), Sections.end()), - const_defined_atom_iterator(Sections.end(), Sections.end())); - } - - /// Returns the atom with the given name, which must exist in this graph. - Atom &getAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); - return *I->second; - } - - /// Returns the atom with the given name, which must exist in this graph and - /// be a DefinedAtom. - DefinedAtom &getDefinedAtomByName(StringRef Name) { - auto &A = getAtomByName(Name); - assert(A.isDefined() && "Atom is not a defined atom"); - return static_cast<DefinedAtom &>(A); - } - - /// Search for the given atom by name. - /// Returns the atom (if found) or an error (if no atom with this name - /// exists). - Expected<Atom &> findAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error<JITLinkError>("No atom named " + Name); - return *I->second; - } - - /// Search for the given defined atom by name. - /// Returns the defined atom (if found) or an error (if no atom with this - /// name exists, or if one exists but is not a defined atom). - Expected<DefinedAtom &> findDefinedAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error<JITLinkError>("No atom named " + Name); - if (!I->second->isDefined()) - return make_error<JITLinkError>("Atom " + Name + - " exists but is not a " - "defined atom"); - return static_cast<DefinedAtom &>(*I->second); - } - - /// Returns the atom covering the given address, or an error if no such atom - /// exists. - /// - /// Returns null if no atom exists at the given address. - DefinedAtom *getAtomByAddress(JITTargetAddress Address) { - refreshAddrToAtomCache(); - - // If there are no defined atoms, bail out early. - if (AddrToAtomCache->empty()) - return nullptr; - - // Find the atom *after* the given address. - auto I = AddrToAtomCache->upper_bound(Address); - - // If this address falls before any known atom, bail out. - if (I == AddrToAtomCache->begin()) - return nullptr; - - // The atom we're looking for is the one before the atom we found. - --I; - - // Otherwise range check the atom that was found. - assert(!I->second->getContent().empty() && "Atom content not set"); - if (Address >= I->second->getAddress() + I->second->getContent().size()) - return nullptr; + const_defined_symbol_iterator(Sections.begin(), Sections.end()), + const_defined_symbol_iterator(Sections.end(), Sections.end())); + } - return I->second; + iterator_range<block_iterator> blocks() { + return make_range(Blocks.begin(), Blocks.end()); } - /// Like getAtomByAddress, but returns an Error if the given address is not - /// covered by an atom, rather than a null pointer. - Expected<DefinedAtom &> findAtomByAddress(JITTargetAddress Address) { - if (auto *DA = getAtomByAddress(Address)) - return *DA; - return make_error<JITLinkError>("No atom at address " + - formatv("{0:x16}", Address)); + /// Turn a defined symbol into an external one. + void makeExternal(Symbol &Sym) { + if (Sym.getAddressable().isAbsolute()) { + assert(AbsoluteSymbols.count(&Sym) && + "Sym is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + } else { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Section &Sec = Sym.getBlock().getSection(); + Sec.removeSymbol(Sym); + } + Sym.makeExternal(createAddressable(false)); + ExternalSymbols.insert(&Sym); } - // Remove the given external atom from the graph. - void removeExternalAtom(Atom &A) { - assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom"); - assert(ExternalAtoms.count(&A) && "A is not in the external atoms set"); - ExternalAtoms.erase(&A); - A.~Atom(); + /// Removes an external symbol. Also removes the underlying Addressable. + void removeExternalSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && !Sym.isAbsolute() && + "Sym is not an external symbol"); + assert(ExternalSymbols.count(&Sym) && "Symbol is not in the externals set"); + ExternalSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); } - /// Remove the given absolute atom from the graph. - void removeAbsoluteAtom(Atom &A) { - assert(A.isAbsolute() && "A is not an absolute atom"); - assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set"); - AbsoluteAtoms.erase(&A); - A.~Atom(); + /// Remove an absolute symbol. Also removes the underlying Addressable. + void removeAbsoluteSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && Sym.isAbsolute() && + "Sym is not an absolute symbol"); + assert(AbsoluteSymbols.count(&Sym) && + "Symbol is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); } - /// Remove the given defined atom from the graph. - void removeDefinedAtom(DefinedAtom &DA) { - if (AddrToAtomCache) { - assert(AddrToAtomCache->count(DA.getAddress()) && - "Cache exists, but does not contain atom"); - AddrToAtomCache->erase(DA.getAddress()); - } - if (DA.hasName()) { - assert(NamedAtoms.count(DA.getName()) && "Named atom not in map"); - NamedAtoms.erase(DA.getName()); - } - DA.getSection().removeAtom(DA); - DA.~DefinedAtom(); + /// Removes defined symbols. Does not remove the underlying block. + void removeDefinedSymbol(Symbol &Sym) { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Sym.getBlock().getSection().removeSymbol(Sym); + destroySymbol(Sym); } - /// Invalidate the atom-to-address map. - void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + /// Remove a block. + void removeBlock(Block &B) { + Blocks.erase(&B); + destroyBlock(B); + } /// Dump the graph. /// @@ -778,87 +895,84 @@ public: std::function<StringRef(Edge::Kind)>()); private: - AddressToAtomMap &getAddrToAtomMap() { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - const AddressToAtomMap &getAddrToAtomMap() const { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - void refreshAddrToAtomCache() const { - if (!AddrToAtomCache) { - AddrToAtomCache = AddressToAtomMap(); - for (auto *DA : defined_atoms()) - (*AddrToAtomCache)[DA->getAddress()] = const_cast<DefinedAtom *>(DA); - } - } - - // Put the BumpPtrAllocator first so that we don't free any of the atoms in - // it until all of their destructors have been run. - BumpPtrAllocator AtomAllocator; + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; std::string Name; unsigned PointerSize; support::endianness Endianness; + BlockSet Blocks; SectionList Sections; - NamedAtomMap NamedAtoms; - ExternalAtomSet ExternalAtoms; - ExternalAtomSet AbsoluteAtoms; - mutable Optional<AddressToAtomMap> AddrToAtomCache; + ExternalSymbolSet ExternalSymbols; + ExternalSymbolSet AbsoluteSymbols; }; -/// A function for mutating AtomGraphs. -using AtomGraphPassFunction = std::function<Error(AtomGraph &)>; +/// A function for mutating LinkGraphs. +using LinkGraphPassFunction = std::function<Error(LinkGraph &)>; -/// A list of atom graph passes. -using AtomGraphPassList = std::vector<AtomGraphPassFunction>; +/// A list of LinkGraph passes. +using LinkGraphPassList = std::vector<LinkGraphPassFunction>; -/// An atom graph pass configuration, consisting of a list of pre-prune, +/// An LinkGraph pass configuration, consisting of a list of pre-prune, /// post-prune, and post-fixup passes. struct PassConfiguration { /// Pre-prune passes. /// /// These passes are called on the graph after it is built, and before any - /// atoms have been pruned. + /// symbols have been pruned. /// - /// Notable use cases: Marking atoms live or should-discard. - AtomGraphPassList PrePrunePasses; + /// Notable use cases: Marking symbols live or should-discard. + LinkGraphPassList PrePrunePasses; /// Post-prune passes. /// - /// These passes are called on the graph after dead and should-discard atoms - /// have been removed, but before fixups are applied. + /// These passes are called on the graph after dead stripping, but before + /// fixups are applied. /// - /// Notable use cases: Building GOT, stub, and TLV atoms. - AtomGraphPassList PostPrunePasses; + /// Notable use cases: Building GOT, stub, and TLV symbols. + LinkGraphPassList PostPrunePasses; /// Post-fixup passes. /// - /// These passes are called on the graph after atom contents has been copied + /// These passes are called on the graph after block contents has been copied /// to working memory, and fixups applied. /// /// Notable use cases: Testing and validation. - AtomGraphPassList PostFixupPasses; + LinkGraphPassList PostFixupPasses; }; /// A map of symbol names to resolved addresses. using AsyncLookupResult = DenseMap<StringRef, JITEvaluatedSymbol>; -/// A function to call with a resolved symbol map (See AsyncLookupResult) or an -/// error if resolution failed. -using JITLinkAsyncLookupContinuation = - std::function<void(Expected<AsyncLookupResult> LR)>; +/// A function object to call with a resolved symbol map (See AsyncLookupResult) +/// or an error if resolution failed. +class JITLinkAsyncLookupContinuation { +public: + virtual ~JITLinkAsyncLookupContinuation() {} + virtual void run(Expected<AsyncLookupResult> LR) = 0; + +private: + virtual void anchor(); +}; + +/// Create a lookup continuation from a function object. +template <typename Continuation> +std::unique_ptr<JITLinkAsyncLookupContinuation> +createLookupContinuation(Continuation Cont) { -/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) -/// for the given symbols, calling the given continuation with either the result -/// (if the lookup succeeds), or an error (if the lookup fails). -using JITLinkAsyncLookupFunction = - std::function<void(const DenseSet<StringRef> &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation)>; + class Impl final : public JITLinkAsyncLookupContinuation { + public: + Impl(Continuation C) : C(std::move(C)) {} + void run(Expected<AsyncLookupResult> LR) override { C(std::move(LR)); } + + private: + Continuation C; + }; + + return std::make_unique<Impl>(std::move(Cont)); +} /// Holds context for a single jitLink invocation. class JITLinkContext { @@ -881,13 +995,13 @@ public: /// lookup continutation which it must call with a result to continue the /// linking process. virtual void lookup(const DenseSet<StringRef> &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) = 0; + std::unique_ptr<JITLinkAsyncLookupContinuation> LC) = 0; - /// Called by JITLink once all defined atoms in the graph have been assigned - /// their final memory locations in the target process. At this point he - /// atom graph can be, inspected to build a symbol table however the atom + /// Called by JITLink once all defined symbols in the graph have been assigned + /// their final memory locations in the target process. At this point the + /// LinkGraph can be inspected to build a symbol table, however the block /// content will not generally have been copied to the target location yet. - virtual void notifyResolved(AtomGraph &G) = 0; + virtual void notifyResolved(LinkGraph &G) = 0; /// Called by JITLink to notify the context that the object has been /// finalized (i.e. emitted to memory and memory permissions set). If all of @@ -904,20 +1018,20 @@ public: /// Returns the mark-live pass to be used for this link. If no pass is /// returned (the default) then the target-specific linker implementation will - /// choose a conservative default (usually marking all atoms live). + /// choose a conservative default (usually marking all symbols live). /// This function is only called if shouldAddDefaultTargetPasses returns true, /// otherwise the JITContext is responsible for adding a mark-live pass in /// modifyPassConfig. - virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const; + virtual LinkGraphPassFunction getMarkLivePass(const Triple &TT) const; /// Called by JITLink to modify the pass pipeline prior to linking. /// The default version performs no modification. virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config); }; -/// Marks all atoms in a graph live. This can be used as a default, conservative -/// mark-live implementation. -Error markAllAtomsLive(AtomGraph &G); +/// Marks all symbols in a graph live. This can be used as a default, +/// conservative mark-live implementation. +Error markAllSymbolsLive(LinkGraph &G); /// Basic JITLink implementation. /// diff --git a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h index 9d0b37fe4a4d..ac5a593bb77b 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -33,20 +33,19 @@ public: class SegmentRequest { public: SegmentRequest() = default; - SegmentRequest(size_t ContentSize, unsigned ContentAlign, - uint64_t ZeroFillSize, unsigned ZeroFillAlign) - : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize), - ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {} + SegmentRequest(uint64_t Alignment, size_t ContentSize, + uint64_t ZeroFillSize) + : Alignment(Alignment), ContentSize(ContentSize), + ZeroFillSize(ZeroFillSize) { + assert(isPowerOf2_32(Alignment) && "Alignment must be power of 2"); + } + uint64_t getAlignment() const { return Alignment; } size_t getContentSize() const { return ContentSize; } - unsigned getContentAlignment() const { return ContentAlign; } uint64_t getZeroFillSize() const { return ZeroFillSize; } - unsigned getZeroFillAlignment() const { return ZeroFillAlign; } - private: + uint64_t Alignment = 0; size_t ContentSize = 0; uint64_t ZeroFillSize = 0; - unsigned ContentAlign = 0; - unsigned ZeroFillAlign = 0; }; using SegmentsRequestMap = DenseMap<unsigned, SegmentRequest>; diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h new file mode 100644 index 000000000000..d70b545fff86 --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -0,0 +1,60 @@ +//===---- MachO_arm64.h - JIT link functions for MachO/arm64 ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/arm64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_arm64_Edges { + +enum MachOARM64RelocationKind : Edge::Kind { + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + GOTPage21, + GOTPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_arm64_Edges + +/// jit-link the given object buffer, which must be a MachO arm64 object file. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx); + +/// Return the string name of the given MachO arm64 edge kind. +StringRef getMachOARM64RelocationKindName(Edge::Kind R); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h index 1d5b586afc32..00a7feb86e83 100644 --- a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h +++ b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -22,6 +22,7 @@ namespace MachO_x86_64_Edges { enum MachOX86RelocationKind : Edge::Kind { Branch32 = Edge::FirstRelocation, + Pointer32, Pointer64, Pointer64Anon, PCRel32, diff --git a/include/llvm/ExecutionEngine/JITSymbol.h b/include/llvm/ExecutionEngine/JITSymbol.h index b14154c5b5e8..c0f1ca4b9876 100644 --- a/include/llvm/ExecutionEngine/JITSymbol.h +++ b/include/llvm/ExecutionEngine/JITSymbol.h @@ -23,6 +23,7 @@ #include <string> #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" @@ -217,7 +218,7 @@ private: /// Represents a symbol in the JIT. class JITSymbol { public: - using GetAddressFtor = std::function<Expected<JITTargetAddress>()>; + using GetAddressFtor = unique_function<Expected<JITTargetAddress>()>; /// Create a 'null' symbol, used to represent a "symbol not found" /// result from a successful (non-erroneous) lookup. @@ -325,7 +326,7 @@ class JITSymbolResolver { public: using LookupSet = std::set<StringRef>; using LookupResult = std::map<StringRef, JITEvaluatedSymbol>; - using OnResolvedFunction = std::function<void(Expected<LookupResult>)>; + using OnResolvedFunction = unique_function<void(Expected<LookupResult>)>; virtual ~JITSymbolResolver() = default; diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 5f593a27cad6..7946b5b7b209 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -26,6 +26,7 @@ #include "llvm/ExecutionEngine/Orc/LazyReexports.h" #include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constant.h" @@ -91,6 +92,8 @@ public: /// Sets the partition function. void setPartitionFunction(PartitionFunction Partition); + /// Sets the ImplSymbolMap + void setImplMap(ImplSymbolMap *Imp); /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; @@ -128,6 +131,7 @@ private: PerDylibResourcesMap DylibResources; PartitionFunction Partition = compileRequested; SymbolLinkagePromoter PromoteSymbols; + ImplSymbolMap *AliaseeImpls = nullptr; }; /// Compile-on-demand layer. @@ -187,7 +191,7 @@ private: std::unique_ptr<ResourceOwner<ResourceT>> wrapOwnership(ResourcePtrT ResourcePtr) { using RO = ResourceOwnerImpl<ResourceT, ResourcePtrT>; - return llvm::make_unique<RO>(std::move(ResourcePtr)); + return std::make_unique<RO>(std::move(ResourcePtr)); } struct LogicalDylib { @@ -440,7 +444,7 @@ private: return Error::success(); // Create the GlobalValues module. - auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(), + auto GVsM = std::make_unique<Module>((SrcM.getName() + ".globals").str(), SrcM.getContext()); GVsM->setDataLayout(DL); @@ -633,7 +637,7 @@ private: NewName += F->getName(); } - auto M = llvm::make_unique<Module>(NewName, SrcM.getContext()); + auto M = std::make_unique<Module>(NewName, SrcM.getContext()); M->setDataLayout(SrcM.getDataLayout()); ValueToValueMapTy VMap; diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 94a5618233e4..4f22a4c38796 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" @@ -51,8 +52,7 @@ using SymbolMap = DenseMap<SymbolStringPtr, JITEvaluatedSymbol>; /// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. using SymbolFlagsMap = DenseMap<SymbolStringPtr, JITSymbolFlags>; -/// A base class for materialization failures that allows the failing -/// symbols to be obtained for logging. +/// A map from JITDylibs to sets of symbols. using SymbolDependenceMap = DenseMap<JITDylib *, SymbolNameSet>; /// A list of (JITDylib*, bool) pairs. @@ -108,7 +108,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); /// Callback to notify client that symbols have been resolved. -using SymbolsResolvedCallback = std::function<void(Expected<SymbolMap>)>; +using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; /// Callback to register the dependencies for a given query. using RegisterDependenciesFunction = @@ -124,13 +124,13 @@ class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { public: static char ID; - FailedToMaterialize(SymbolNameSet Symbols); + FailedToMaterialize(std::shared_ptr<SymbolDependenceMap> Symbols); std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; - const SymbolNameSet &getSymbols() const { return Symbols; } + const SymbolDependenceMap &getSymbols() const { return *Symbols; } private: - SymbolNameSet Symbols; + std::shared_ptr<SymbolDependenceMap> Symbols; }; /// Used to notify clients when symbols can not be found during a lookup. @@ -205,12 +205,26 @@ public: /// symbols must be ones covered by this MaterializationResponsibility /// instance. Individual calls to this method may resolve a subset of the /// symbols, but all symbols must have been resolved prior to calling emit. - void notifyResolved(const SymbolMap &Symbols); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyResolved(const SymbolMap &Symbols); /// Notifies the target JITDylib (and any pending queries on that JITDylib) /// that all symbols covered by this MaterializationResponsibility instance /// have been emitted. - void notifyEmitted(); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyEmitted(); /// Adds new symbols to the JITDylib and this responsibility instance. /// JITDylib entries start out in the materializing state. @@ -346,7 +360,7 @@ private: /// inline std::unique_ptr<AbsoluteSymbolsMaterializationUnit> absoluteSymbols(SymbolMap Symbols, VModuleKey K = VModuleKey()) { - return llvm::make_unique<AbsoluteSymbolsMaterializationUnit>( + return std::make_unique<AbsoluteSymbolsMaterializationUnit>( std::move(Symbols), std::move(K)); } @@ -390,7 +404,7 @@ private: /// \endcode inline std::unique_ptr<ReExportsMaterializationUnit> symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( nullptr, true, std::move(Aliases), std::move(K)); } @@ -402,7 +416,7 @@ symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { inline std::unique_ptr<ReExportsMaterializationUnit> reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, bool MatchNonExported = false, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( &SourceJD, MatchNonExported, std::move(Aliases), std::move(K)); } @@ -411,32 +425,13 @@ reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); -/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically -/// re-export a subset of the source JITDylib's symbols in the target. -class ReexportsGenerator { -public: - using SymbolPredicate = std::function<bool(SymbolStringPtr)>; - - /// Create a reexports generator. If an Allow predicate is passed, only - /// symbols for which the predicate returns true will be reexported. If no - /// Allow predicate is passed, all symbols will be exported. - ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, - SymbolPredicate Allow = SymbolPredicate()); - - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); - -private: - JITDylib &SourceJD; - bool MatchNonExported = false; - SymbolPredicate Allow; -}; - /// Represents the state that a symbol has reached during materialization. enum class SymbolState : uint8_t { Invalid, /// No symbol should be in this state. NeverSearched, /// Added to the symbol table, never queried. Materializing, /// Queried, materialization begun. Resolved, /// Assigned address, still materializing. + Emitted, /// Emitted to memory, but waiting on transitive dependencies. Ready = 0x3f /// Ready and safe for clients to access. }; @@ -502,8 +497,12 @@ class JITDylib { friend class ExecutionSession; friend class MaterializationResponsibility; public: - using GeneratorFunction = std::function<Expected<SymbolNameSet>( - JITDylib &Parent, const SymbolNameSet &Names)>; + class DefinitionGenerator { + public: + virtual ~DefinitionGenerator(); + virtual Expected<SymbolNameSet> + tryToGenerate(JITDylib &Parent, const SymbolNameSet &Names) = 0; + }; using AsynchronousSymbolQuerySet = std::set<std::shared_ptr<AsynchronousSymbolQuery>>; @@ -519,13 +518,20 @@ public: /// Get a reference to the ExecutionSession for this JITDylib. ExecutionSession &getExecutionSession() const { return ES; } - /// Set a definition generator. If set, whenever a symbol fails to resolve - /// within this JITDylib, lookup and lookupFlags will pass the unresolved - /// symbols set to the definition generator. The generator can optionally - /// add a definition for the unresolved symbols to the dylib. - void setGenerator(GeneratorFunction DefGenerator) { - this->DefGenerator = std::move(DefGenerator); - } + /// Adds a definition generator to this JITDylib and returns a referenece to + /// it. + /// + /// When JITDylibs are searched during lookup, if no existing definition of + /// a symbol is found, then any generators that have been added are run (in + /// the order that they were added) to potentially generate a definition. + template <typename GeneratorT> + GeneratorT &addGenerator(std::unique_ptr<GeneratorT> DefGenerator); + + /// Remove a definition generator from this JITDylib. + /// + /// The given generator must exist in this JITDylib's generators list (i.e. + /// have been added and not yet removed). + void removeGenerator(DefinitionGenerator &G); /// Set the search order to be used when fixing up definitions in JITDylib. /// This will replace the previous search order, and apply to any symbol @@ -633,17 +639,17 @@ private: struct MaterializingInfo { SymbolDependenceMap Dependants; SymbolDependenceMap UnemittedDependencies; - bool IsEmitted = false; void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q); void removeQuery(const AsynchronousSymbolQuery &Q); AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState); - AsynchronousSymbolQueryList takeAllQueries(); + AsynchronousSymbolQueryList takeAllPendingQueries() { + return std::move(PendingQueries); + } bool hasQueriesPending() const { return !PendingQueries.empty(); } const AsynchronousSymbolQueryList &pendingQueries() const { return PendingQueries; } - private: AsynchronousSymbolQueryList PendingQueries; }; @@ -710,9 +716,9 @@ private: SymbolNameSet &Unresolved, bool MatchNonExported, MaterializationUnitList &MUs); - void lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs); bool lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, std::vector<std::unique_ptr<MaterializationUnit>> &MUs, @@ -734,18 +740,20 @@ private: void addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependants); - void resolve(const SymbolMap &Resolved); + Error resolve(const SymbolMap &Resolved); - void emit(const SymbolFlagsMap &Emitted); + Error emit(const SymbolFlagsMap &Emitted); - void notifyFailed(const SymbolNameSet &FailedSymbols); + using FailedSymbolsWorklist = + std::vector<std::pair<JITDylib *, SymbolStringPtr>>; + static void notifyFailed(FailedSymbolsWorklist FailedSymbols); ExecutionSession &ES; std::string JITDylibName; SymbolTable Symbols; UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; - GeneratorFunction DefGenerator; + std::vector<std::unique_ptr<DefinitionGenerator>> DefGenerators; JITDylibSearchList SearchOrder; }; @@ -933,6 +941,14 @@ private: OutstandingMUs; }; +template <typename GeneratorT> +GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { + auto &G = *DefGenerator; + ES.runSessionLocked( + [&]() { DefGenerators.push_back(std::move(DefGenerator)); }); + return G; +} + template <typename Func> auto JITDylib::withSearchOrderDo(Func &&F) -> decltype(F(std::declval<const JITDylibSearchList &>())) { @@ -972,6 +988,27 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) { }); } +/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically +/// re-export a subset of the source JITDylib's symbols in the target. +class ReexportsGenerator : public JITDylib::DefinitionGenerator { +public: + using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + + /// Create a reexports generator. If an Allow predicate is passed, only + /// symbols for which the predicate returns true will be reexported. If no + /// Allow predicate is passed, all symbols will be exported. + ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, + SymbolPredicate Allow = SymbolPredicate()); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + JITDylib &SourceJD; + bool MatchNonExported = false; + SymbolPredicate Allow; +}; + /// Mangles symbol names then uniques them in the context of an /// ExecutionSession. class MangleAndInterner { diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 75865920c741..cf0a428662ef 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -19,6 +19,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Object/Archive.h" #include "llvm/Support/DynamicLibrary.h" #include <algorithm> #include <cstdint> @@ -37,6 +38,8 @@ class Value; namespace orc { +class ObjectLayer; + /// This iterator provides a convenient way to iterate over the elements /// of an llvm.global_ctors/llvm.global_dtors instance. /// @@ -237,7 +240,7 @@ public: /// If an instance of this class is attached to a JITDylib as a fallback /// definition generator, then any symbol found in the given DynamicLibrary that /// passes the 'Allow' predicate will be added to the JITDylib. -class DynamicLibrarySearchGenerator { +class DynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator { public: using SymbolPredicate = std::function<bool(SymbolStringPtr)>; @@ -253,19 +256,20 @@ public: /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> GetForCurrentProcess(char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()) { return Load(nullptr, GlobalPrefix, std::move(Allow)); } - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; private: sys::DynamicLibrary Dylib; @@ -273,6 +277,40 @@ private: char GlobalPrefix; }; +/// A utility class to expose symbols from a static library. +/// +/// If an instance of this class is attached to a JITDylib as a fallback +/// definition generator, then any symbol found in the archive will result in +/// the containing object being added to the JITDylib. +class StaticLibraryDefinitionGenerator : public JITDylib::DefinitionGenerator { +public: + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// is a valid archive, otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName); + + /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. + /// Thhis call will succeed if the buffer contains a valid archive, otherwise + /// it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + StaticLibraryDefinitionGenerator(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> ArchiveBuffer, + Error &Err); + + ObjectLayer &L; + std::unique_ptr<MemoryBuffer> ArchiveBuffer; + object::Archive Archive; + size_t UnrealizedObjects = 0; +}; + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index 1b4c8b6cd95f..b71e5b339711 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -22,6 +22,9 @@ namespace llvm { class Module; namespace orc { +/// A layer that applies a transform to emitted modules. +/// The transform function is responsible for locking the ThreadSafeContext +/// before operating on the module. class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function<Expected<ThreadSafeModule>( diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h index 0aac1916423f..b1e47d77557c 100644 --- a/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -184,8 +184,8 @@ private: class LLJITBuilderState { public: - using ObjectLinkingLayerCreator = - std::function<std::unique_ptr<ObjectLayer>(ExecutionSession &)>; + using ObjectLinkingLayerCreator = std::function<std::unique_ptr<ObjectLayer>( + ExecutionSession &, const Triple &TT)>; using CompileFunctionCreator = std::function<Expected<IRCompileLayer::CompileFunction>( diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 855e31b33549..b31914f12a0d 100644 --- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -16,6 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/OrcV1Deprecation.h" #include <memory> namespace llvm { @@ -62,7 +63,7 @@ std::shared_ptr<LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>> createLambdaResolver(DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(std::move(DylibLookupFtor), + return std::make_unique<LR>(std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } @@ -72,7 +73,7 @@ createLambdaResolver(ORCv1DeprecationAcknowledgement, DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(AcknowledgeORCv1Deprecation, + return std::make_unique<LR>(AcknowledgeORCv1Deprecation, std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 16202d89f861..b67a9feed523 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -49,28 +49,24 @@ private: switch (EmitState) { case NotEmitted: if (auto GV = searchGVs(Name, ExportedSymbolsOnly)) { - // Create a std::string version of Name to capture here - the argument - // (a StringRef) may go away before the lambda is executed. - // FIXME: Use capture-init when we move to C++14. - std::string PName = Name; JITSymbolFlags Flags = JITSymbolFlags::fromGlobalValue(*GV); - auto GetAddress = - [this, ExportedSymbolsOnly, PName, &B]() -> Expected<JITTargetAddress> { - if (this->EmitState == Emitting) - return 0; - else if (this->EmitState == NotEmitted) { - this->EmitState = Emitting; - if (auto Err = this->emitToBaseLayer(B)) - return std::move(Err); - this->EmitState = Emitted; - } - if (auto Sym = B.findSymbolIn(K, PName, ExportedSymbolsOnly)) - return Sym.getAddress(); - else if (auto Err = Sym.takeError()) + auto GetAddress = [this, ExportedSymbolsOnly, Name = Name.str(), + &B]() -> Expected<JITTargetAddress> { + if (this->EmitState == Emitting) + return 0; + else if (this->EmitState == NotEmitted) { + this->EmitState = Emitting; + if (auto Err = this->emitToBaseLayer(B)) return std::move(Err); - else - llvm_unreachable("Successful symbol lookup should return " - "definition address here"); + this->EmitState = Emitted; + } + if (auto Sym = B.findSymbolIn(K, Name, ExportedSymbolsOnly)) + return Sym.getAddress(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + else + llvm_unreachable("Successful symbol lookup should return " + "definition address here"); }; return JITSymbol(std::move(GetAddress), Flags); } else @@ -171,7 +167,7 @@ private: bool ExportedSymbolsOnly) const { assert(!MangledSymbols && "Mangled symbols map already exists?"); - auto Symbols = llvm::make_unique<StringMap<const GlobalValue*>>(); + auto Symbols = std::make_unique<StringMap<const GlobalValue*>>(); Mangler Mang; @@ -209,7 +205,7 @@ public: Error addModule(VModuleKey K, std::unique_ptr<Module> M) { assert(!ModuleMap.count(K) && "VModuleKey K already in use"); ModuleMap[K] = - llvm::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); + std::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); return Error::success(); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/include/llvm/ExecutionEngine/Orc/LazyReexports.h index 9fdd1d15f782..311ed59b1549 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -18,6 +18,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" namespace llvm { @@ -70,7 +71,7 @@ public: template <typename NotifyResolvedImpl> static std::unique_ptr<NotifyResolvedFunction> createNotifyResolvedFunction(NotifyResolvedImpl NotifyResolved) { - return llvm::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( + return std::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( std::move(NotifyResolved)); } @@ -159,7 +160,7 @@ public: IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, - VModuleKey K); + ImplSymbolMap *SrcJDLoc, VModuleKey K); StringRef getName() const override; @@ -174,6 +175,7 @@ private: SymbolAliasMap CallableAliases; std::shared_ptr<LazyCallThroughManager::NotifyResolvedFunction> NotifyResolved; + ImplSymbolMap *AliaseeTable; }; /// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export @@ -182,9 +184,10 @@ private: inline std::unique_ptr<LazyReexportsMaterializationUnit> lazyReexports(LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, - SymbolAliasMap CallableAliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<LazyReexportsMaterializationUnit>( - LCTManager, ISManager, SourceJD, std::move(CallableAliases), + SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc = nullptr, + VModuleKey K = VModuleKey()) { + return std::make_unique<LazyReexportsMaterializationUnit>( + LCTManager, ISManager, SourceJD, std::move(CallableAliases), SrcJDLoc, std::move(K)); } diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h index f9cbbf6ff180..148e260c9569 100644 --- a/include/llvm/ExecutionEngine/Orc/Legacy.h +++ b/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -84,7 +84,7 @@ createSymbolResolver(GetResponsibilitySetFn &&GetResponsibilitySet, typename std::remove_reference<GetResponsibilitySetFn>::type>::type, typename std::remove_cv< typename std::remove_reference<LookupFn>::type>::type>; - return llvm::make_unique<LambdaSymbolResolverImpl>( + return std::make_unique<LambdaSymbolResolverImpl>( std::forward<GetResponsibilitySetFn>(GetResponsibilitySet), std::forward<LookupFn>(Lookup)); } diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index c1e7d27f446e..caf8e707516d 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -73,6 +73,9 @@ public: virtual Error notifyRemovingAllModules() { return Error::success(); } }; + using ReturnObjectBufferFunction = + std::function<void(std::unique_ptr<MemoryBuffer>)>; + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyEmitted functors. ObjectLinkingLayer(ExecutionSession &ES, @@ -81,6 +84,13 @@ public: /// Destruct an ObjectLinkingLayer. ~ObjectLinkingLayer(); + /// Set an object buffer return function. By default object buffers are + /// deleted once the JIT has linked them. If a return function is set then + /// it will be called to transfer ownership of the buffer instead. + void setReturnObjectBuffer(ReturnObjectBufferFunction ReturnObjectBuffer) { + this->ReturnObjectBuffer = std::move(ReturnObjectBuffer); + } + /// Add a pass-config modifier. ObjectLinkingLayer &addPlugin(std::unique_ptr<Plugin> P) { std::lock_guard<std::mutex> Lock(LayerMutex); @@ -138,6 +148,7 @@ private: jitlink::JITLinkMemoryManager &MemMgr; bool OverrideObjectFlags = false; bool AutoClaimObjectSymbols = false; + ReturnObjectBufferFunction ReturnObjectBuffer; DenseMap<VModuleKey, AllocPtr> TrackedAllocs; std::vector<AllocPtr> UntrackedAllocs; std::vector<std::unique_ptr<Plugin>> Plugins; @@ -153,10 +164,16 @@ public: Error notifyRemovingAllModules() override; private: + + struct EHFrameRange { + JITTargetAddress Addr = 0; + size_t Size; + }; + jitlink::EHFrameRegistrar &Registrar; - DenseMap<MaterializationResponsibility *, JITTargetAddress> InProcessLinks; - DenseMap<VModuleKey, JITTargetAddress> TrackedEHFrameAddrs; - std::vector<JITTargetAddress> UntrackedEHFrameAddrs; + DenseMap<MaterializationResponsibility *, EHFrameRange> InProcessLinks; + DenseMap<VModuleKey, EHFrameRange> TrackedEHFrameRanges; + std::vector<EHFrameRange> UntrackedEHFrameRanges; }; } // end namespace orc diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 8b875b7906e1..86e8d5df3ad9 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -493,7 +493,7 @@ public: ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : JITCompileCallbackManager( - llvm::make_unique<RemoteTrampolinePool>(Client), ES, + std::make_unique<RemoteTrampolinePool>(Client), ES, ErrorHandlerAddress) {} }; @@ -553,7 +553,7 @@ public: auto Id = IndirectStubOwnerIds.getNext(); if (auto Err = callB<stubs::CreateIndirectStubsOwner>(Id)) return std::move(Err); - return llvm::make_unique<RemoteIndirectStubsManager>(*this, Id); + return std::make_unique<RemoteIndirectStubsManager>(*this, Id); } Expected<RemoteCompileCallbackManager &> diff --git a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h index 07c7471afc6a..752a0a34e0a1 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h @@ -359,9 +359,9 @@ public: { assert(KeyName != nullptr && "No keyname pointer"); std::lock_guard<std::recursive_mutex> Lock(SerializersMutex); - // FIXME: Move capture Serialize once we have C++14. Serializers[ErrorInfoT::classID()] = - [KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error { + [KeyName, Serialize = std::move(Serialize)]( + ChannelT &C, const ErrorInfoBase &EIB) -> Error { assert(EIB.dynamicClassID() == ErrorInfoT::classID() && "Serializer called for wrong error type"); if (auto Err = serializeSeq(C, *KeyName)) @@ -551,26 +551,26 @@ public: /// RPC channel serialization for std::tuple. static Error serialize(ChannelT &C, const std::tuple<ArgTs...> &V) { - return serializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return serializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } /// RPC channel deserialization for std::tuple. static Error deserialize(ChannelT &C, std::tuple<ArgTs...> &V) { - return deserializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return deserializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } private: // Serialization helper for std::tuple. template <size_t... Is> static Error serializeTupleHelper(ChannelT &C, const std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return serializeSeq(C, std::get<Is>(V)...); } // Serialization helper for std::tuple. template <size_t... Is> static Error deserializeTupleHelper(ChannelT &C, std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return deserializeSeq(C, std::get<Is>(V)...); } }; diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h index 3b11e1b283de..ee9c2cc69c30 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -338,7 +338,9 @@ public: return Err; // Close the response message. - return C.endSendMessage(); + if (auto Err = C.endSendMessage()) + return Err; + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -350,7 +352,9 @@ public: return Err2; if (auto Err2 = serializeSeq(C, std::move(Err))) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -378,8 +382,11 @@ public: C, *ResultOrErr)) return Err; - // Close the response message. - return C.endSendMessage(); + // End the response message. + if (auto Err = C.endSendMessage()) + return Err; + + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -389,7 +396,9 @@ public: return Err; if (auto Err2 = C.startSendMessage(ResponseId, SeqNo)) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -502,7 +511,7 @@ public: static typename WrappedHandlerReturn<RetT>::Type unpackAndRun(HandlerT &Handler, std::tuple<TArgTs...> &Args) { return unpackAndRunHelper(Handler, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -510,7 +519,7 @@ public: static Error unpackAndRunAsync(HandlerT &Handler, ResponderT &Responder, std::tuple<TArgTs...> &Args) { return unpackAndRunAsyncHelper(Handler, Responder, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -540,14 +549,13 @@ public: // Deserialize arguments from the channel. template <typename ChannelT, typename... CArgTs> static Error deserializeArgs(ChannelT &C, std::tuple<CArgTs...> &Args) { - return deserializeArgsHelper(C, Args, - llvm::index_sequence_for<CArgTs...>()); + return deserializeArgsHelper(C, Args, std::index_sequence_for<CArgTs...>()); } private: template <typename ChannelT, typename... CArgTs, size_t... Indexes> static Error deserializeArgsHelper(ChannelT &C, std::tuple<CArgTs...> &Args, - llvm::index_sequence<Indexes...> _) { + std::index_sequence<Indexes...> _) { return SequenceSerialization<ChannelT, ArgTs...>::deserialize( C, std::get<Indexes>(Args)...); } @@ -556,18 +564,16 @@ private: static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunHelper(HandlerT &Handler, ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + std::index_sequence<Indexes...>) { return run(Handler, std::move(std::get<Indexes>(Args))...); } - template <typename HandlerT, typename ResponderT, typename ArgTuple, size_t... Indexes> static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunAsyncHelper(HandlerT &Handler, ResponderT &Responder, - ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + ArgTuple &Args, std::index_sequence<Indexes...>) { return run(Handler, Responder, std::move(std::get<Indexes>(Args))...); } }; @@ -743,11 +749,15 @@ public: // to the user defined handler. Error handleResponse(ChannelT &C) override { Error Result = Error::success(); - if (auto Err = - SerializationTraits<ChannelT, Error, Error>::deserialize(C, Result)) + if (auto Err = SerializationTraits<ChannelT, Error, Error>::deserialize( + C, Result)) { + consumeError(std::move(Result)); return Err; - if (auto Err = C.endReceiveMessage()) + } + if (auto Err = C.endReceiveMessage()) { + consumeError(std::move(Result)); return Err; + } return Handler(std::move(Result)); } @@ -767,7 +777,7 @@ private: // Create a ResponseHandler from a given user handler. template <typename ChannelT, typename FuncRetT, typename HandlerT> std::unique_ptr<ResponseHandler<ChannelT>> createResponseHandler(HandlerT H) { - return llvm::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( + return std::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( std::move(H)); } @@ -1403,14 +1413,12 @@ public: using ErrorReturn = typename RTraits::ErrorReturnType; using ErrorReturnPromise = typename RTraits::ReturnPromiseType; - // FIXME: Stack allocate and move this into the handler once LLVM builds - // with C++14. - auto Promise = std::make_shared<ErrorReturnPromise>(); - auto FutureResult = Promise->get_future(); + ErrorReturnPromise Promise; + auto FutureResult = Promise.get_future(); if (auto Err = this->template appendCallAsync<Func>( - [Promise](ErrorReturn RetOrErr) { - Promise->set_value(std::move(RetOrErr)); + [Promise = std::move(Promise)](ErrorReturn RetOrErr) mutable { + Promise.set_value(std::move(RetOrErr)); return Error::success(); }, Args...)) { @@ -1523,6 +1531,12 @@ public: return std::move(Err); } + if (auto Err = this->C.send()) { + detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( + std::move(Result)); + return std::move(Err); + } + while (!ReceivedResponse) { if (auto Err = this->handleOne()) { detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( @@ -1582,8 +1596,7 @@ public: // outstanding calls count, then poke the condition variable. using ArgType = typename detail::ResponseHandlerArg< typename detail::HandlerTraits<HandlerT>::Type>::ArgType; - // FIXME: Move handler into wrapped handler once we have C++14. - auto WrappedHandler = [this, Handler](ArgType Arg) { + auto WrappedHandler = [this, Handler = std::move(Handler)](ArgType Arg) { auto Err = Handler(std::move(Arg)); std::unique_lock<std::mutex> Lock(M); --NumOutstandingCalls; diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index d9535ce5f21f..c5106cf09ecc 100644 --- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -216,7 +216,7 @@ private: : K(std::move(K)), Parent(Parent), MemMgr(std::move(MemMgr)), - PFC(llvm::make_unique<PreFinalizeContents>( + PFC(std::make_unique<PreFinalizeContents>( std::move(Obj), std::move(Resolver), ProcessAllSections)) { buildInitialSymbolTable(PFC->Obj); @@ -234,7 +234,7 @@ private: JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver, nullptr); - PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); + PFC->RTDyld = std::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections); Finalized = true; @@ -338,7 +338,7 @@ private: std::shared_ptr<SymbolResolver> Resolver, bool ProcessAllSections) { using LOS = ConcreteLinkedObject<MemoryManagerPtrT>; - return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj), + return std::make_unique<LOS>(Parent, std::move(K), std::move(Obj), std::move(MemMgr), std::move(Resolver), ProcessAllSections); } diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h index b87cf697a81e..d7304cfcf931 100644 --- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -137,17 +137,12 @@ protected: RemoteSymbolId Id) : C(C), Id(Id) {} - RemoteSymbolMaterializer(const RemoteSymbolMaterializer &Other) - : C(Other.C), Id(Other.Id) { - // FIXME: This is a horrible, auto_ptr-style, copy-as-move operation. - // It should be removed as soon as LLVM has C++14's generalized - // lambda capture (at which point the materializer can be moved - // into the lambda in remoteToJITSymbol below). - const_cast<RemoteSymbolMaterializer&>(Other).Id = 0; + RemoteSymbolMaterializer(RemoteSymbolMaterializer &&Other) + : C(Other.C), Id(Other.Id) { + Other.Id = 0; } - RemoteSymbolMaterializer& - operator=(const RemoteSymbolMaterializer&) = delete; + RemoteSymbolMaterializer &operator=(RemoteSymbolMaterializer &&) = delete; /// Release the remote symbol. ~RemoteSymbolMaterializer() { @@ -218,9 +213,9 @@ protected: return nullptr; // else... RemoteSymbolMaterializer RSM(*this, RemoteSym.first); - auto Sym = - JITSymbol([RSM]() mutable { return RSM.materialize(); }, - RemoteSym.second); + auto Sym = JITSymbol( + [RSM = std::move(RSM)]() mutable { return RSM.materialize(); }, + RemoteSym.second); return Sym; } else return RemoteSymOrErr.takeError(); @@ -472,7 +467,7 @@ private: } Expected<ObjHandleT> addObject(std::string ObjBuffer) { - auto Buffer = llvm::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); + auto Buffer = std::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); auto Id = HandleIdMgr.getNext(); assert(!BaseLayerHandles.count(Id) && "Id already in use?"); diff --git a/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h new file mode 100644 index 000000000000..cf57b63b6448 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h @@ -0,0 +1,84 @@ +//===-- SpeculateAnalyses.h --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// \file +/// Contains the Analyses and Result Interpretation to select likely functions +/// to Speculatively compile before they are called. [Purely Experimentation] +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H + +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" + +#include <vector> + +namespace llvm { + +namespace orc { + +// Provides common code. +class SpeculateQuery { +protected: + void findCalles(const BasicBlock *, DenseSet<StringRef> &); + bool isStraightLine(const Function &F); + +public: + using ResultTy = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; +}; + +// Direct calls in high frequency basic blocks are extracted. +class BlockFreqQuery : public SpeculateQuery { + size_t numBBToGet(size_t); + +public: + // Find likely next executables based on IR Block Frequency + ResultTy operator()(Function &F); +}; + +// This Query generates a sequence of basic blocks which follows the order of +// execution. +// A handful of BB with higher block frequencies are taken, then path to entry +// and end BB are discovered by traversing up & down the CFG. +class SequenceBBQuery : public SpeculateQuery { + struct WalkDirection { + bool Upward = true, Downward = true; + // the block associated contain a call + bool CallerBlock = false; + }; + +public: + using VisitedBlocksInfoTy = DenseMap<const BasicBlock *, WalkDirection>; + using BlockListTy = SmallVector<const BasicBlock *, 8>; + using BackEdgesInfoTy = + SmallVector<std::pair<const BasicBlock *, const BasicBlock *>, 8>; + using BlockFreqInfoTy = + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8>; + +private: + std::size_t getHottestBlocks(std::size_t TotalBlocks); + BlockListTy rearrangeBB(const Function &, const BlockListTy &); + BlockListTy queryCFG(Function &, const BlockListTy &); + void traverseToEntryBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + void traverseToExitBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + +public: + ResultTy operator()(Function &F); +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H diff --git a/include/llvm/ExecutionEngine/Orc/Speculation.h b/include/llvm/ExecutionEngine/Orc/Speculation.h new file mode 100644 index 000000000000..766a6b070f12 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -0,0 +1,207 @@ +//===-- Speculation.h - Speculative Compilation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition to support speculative compilation when laziness is +// enabled. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATION_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/Debug.h" + +#include <mutex> +#include <type_traits> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { + +class Speculator; + +// Track the Impls (JITDylib,Symbols) of Symbols while lazy call through +// trampolines are created. Operations are guarded by locks tp ensure that Imap +// stays in consistent state after read/write + +class ImplSymbolMap { + friend class Speculator; + +public: + using AliaseeDetails = std::pair<SymbolStringPtr, JITDylib *>; + using Alias = SymbolStringPtr; + using ImapTy = DenseMap<Alias, AliaseeDetails>; + void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD); + +private: + // FIX ME: find a right way to distinguish the pre-compile Symbols, and update + // the callsite + Optional<AliaseeDetails> getImplFor(const SymbolStringPtr &StubSymbol) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto Position = Maps.find(StubSymbol); + if (Position != Maps.end()) + return Position->getSecond(); + else + return None; + } + + std::mutex ConcurrentAccess; + ImapTy Maps; +}; + +// Defines Speculator Concept, +class Speculator { +public: + using TargetFAddr = JITTargetAddress; + using FunctionCandidatesMap = DenseMap<SymbolStringPtr, SymbolNameSet>; + using StubAddrLikelies = DenseMap<TargetFAddr, SymbolNameSet>; + +private: + void registerSymbolsWithAddr(TargetFAddr ImplAddr, + SymbolNameSet likelySymbols) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + GlobalSpecMap.insert({ImplAddr, std::move(likelySymbols)}); + } + + void launchCompile(JITTargetAddress FAddr) { + SymbolNameSet CandidateSet; + // Copy CandidateSet is necessary, to avoid unsynchronized access to + // the datastructure. + { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto It = GlobalSpecMap.find(FAddr); + if (It == GlobalSpecMap.end()) + return; + CandidateSet = It->getSecond(); + } + + SymbolDependenceMap SpeculativeLookUpImpls; + + for (auto &Callee : CandidateSet) { + auto ImplSymbol = AliaseeImplTable.getImplFor(Callee); + // try to distinguish already compiled & library symbols + if (!ImplSymbol.hasValue()) + continue; + const auto &ImplSymbolName = ImplSymbol.getPointer()->first; + JITDylib *ImplJD = ImplSymbol.getPointer()->second; + auto &SymbolsInJD = SpeculativeLookUpImpls[ImplJD]; + SymbolsInJD.insert(ImplSymbolName); + } + + DEBUG_WITH_TYPE("orc", for (auto &I + : SpeculativeLookUpImpls) { + llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; + for (auto &N : I.second) + llvm::dbgs() << "\n Likely Symbol : " << N; + }); + + // for a given symbol, there may be no symbol qualified for speculatively + // compile try to fix this before jumping to this code if possible. + for (auto &LookupPair : SpeculativeLookUpImpls) + ES.lookup(JITDylibSearchList({{LookupPair.first, true}}), + LookupPair.second, SymbolState::Ready, + [this](Expected<SymbolMap> Result) { + if (auto Err = Result.takeError()) + ES.reportError(std::move(Err)); + }, + NoDependenciesToRegister); + } + +public: + Speculator(ImplSymbolMap &Impl, ExecutionSession &ref) + : AliaseeImplTable(Impl), ES(ref), GlobalSpecMap(0) {} + Speculator(const Speculator &) = delete; + Speculator(Speculator &&) = delete; + Speculator &operator=(const Speculator &) = delete; + Speculator &operator=(Speculator &&) = delete; + + /// Define symbols for this Speculator object (__orc_speculator) and the + /// speculation runtime entry point symbol (__orc_speculate_for) in the + /// given JITDylib. + Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle); + + // Speculatively compile likely functions for the given Stub Address. + // destination of __orc_speculate_for jump + void speculateFor(TargetFAddr StubAddr) { launchCompile(StubAddr); } + + // FIXME : Register with Stub Address, after JITLink Fix. + void registerSymbols(FunctionCandidatesMap Candidates, JITDylib *JD) { + for (auto &SymPair : Candidates) { + auto Target = SymPair.first; + auto Likely = SymPair.second; + + auto OnReadyFixUp = [Likely, Target, + this](Expected<SymbolMap> ReadySymbol) { + if (ReadySymbol) { + auto RAddr = (*ReadySymbol)[Target].getAddress(); + registerSymbolsWithAddr(RAddr, std::move(Likely)); + } else + this->getES().reportError(ReadySymbol.takeError()); + }; + // Include non-exported symbols also. + ES.lookup(JITDylibSearchList({{JD, true}}), SymbolNameSet({Target}), + SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); + } + } + + ExecutionSession &getES() { return ES; } + +private: + static void speculateForEntryPoint(Speculator *Ptr, uint64_t StubId); + std::mutex ConcurrentAccess; + ImplSymbolMap &AliaseeImplTable; + ExecutionSession &ES; + StubAddrLikelies GlobalSpecMap; +}; + +class IRSpeculationLayer : public IRLayer { +public: + using IRlikiesStrRef = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; + using ResultEval = std::function<IRlikiesStrRef(Function &)>; + using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>; + + IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, + Speculator &Spec, MangleAndInterner &Mangle, + ResultEval Interpreter) + : IRLayer(ES), NextLayer(BaseLayer), S(Spec), Mangle(Mangle), + QueryAnalysis(Interpreter) {} + + void emit(MaterializationResponsibility R, ThreadSafeModule TSM); + +private: + TargetAndLikelies + internToJITSymbols(DenseMap<StringRef, DenseSet<StringRef>> IRNames) { + assert(!IRNames.empty() && "No IRNames received to Intern?"); + TargetAndLikelies InternedNames; + DenseSet<SymbolStringPtr> TargetJITNames; + for (auto &NamePair : IRNames) { + for (auto &TargetNames : NamePair.second) + TargetJITNames.insert(Mangle(TargetNames)); + + InternedNames[Mangle(NamePair.first)] = std::move(TargetJITNames); + } + return InternedNames; + } + + IRCompileLayer &NextLayer; + Speculator &S; + MangleAndInterner &Mangle; + ResultEval QueryAnalysis; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATION_H diff --git a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h index 5787500387c4..2347faed37a2 100644 --- a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h +++ b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -38,17 +38,12 @@ private: public: // RAII based lock for ThreadSafeContext. class LLVM_NODISCARD Lock { - private: - using UnderlyingLock = std::lock_guard<std::recursive_mutex>; - public: - Lock(std::shared_ptr<State> S) - : S(std::move(S)), - L(llvm::make_unique<UnderlyingLock>(this->S->Mutex)) {} + Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {} private: std::shared_ptr<State> S; - std::unique_ptr<UnderlyingLock> L; + std::unique_lock<std::recursive_mutex> L; }; /// Construct a null context. @@ -69,7 +64,7 @@ public: /// instance, or null if the instance was default constructed. const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; } - Lock getLock() { + Lock getLock() const { assert(S && "Can not lock an empty ThreadSafeContext"); return Lock(S); } @@ -95,7 +90,7 @@ public: // We also need to lock the context to make sure the module tear-down // does not overlap any other work on the context. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } M = std::move(Other.M); @@ -117,23 +112,14 @@ public: ~ThreadSafeModule() { // We need to lock the context while we destruct the module. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } } - /// Get the module wrapped by this ThreadSafeModule. - Module *getModule() { return M.get(); } - - /// Get the module wrapped by this ThreadSafeModule. - const Module *getModule() const { return M.get(); } - - /// Take out a lock on the ThreadSafeContext for this module. - ThreadSafeContext::Lock getContextLock() { return TSCtx.getLock(); } - /// Boolean conversion: This ThreadSafeModule will evaluate to true if it /// wraps a non-null module. - explicit operator bool() { + explicit operator bool() const { if (M) { assert(TSCtx.getContext() && "Non-null module must have non-null context"); @@ -142,6 +128,33 @@ public: return false; } + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) -> decltype(F(std::declval<Module &>())) { + assert(M && "Can not call on null module"); + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) const + -> decltype(F(std::declval<const Module &>())) { + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Get a raw pointer to the contained module without locking the context. + Module *getModuleUnlocked() { return M.get(); } + + /// Get a raw pointer to the contained module without locking the context. + const Module *getModuleUnlocked() const { return M.get(); } + + /// Returns the context for this ThreadSafeModule. + ThreadSafeContext getContext() const { return TSCtx; } + private: std::unique_ptr<Module> M; ThreadSafeContext TSCtx; diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h index b2b4eba47074..ce7024a7f19b 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -13,6 +13,7 @@ #ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H #define LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/DIContext.h" @@ -271,10 +272,10 @@ private: std::unique_ptr<MemoryBuffer> UnderlyingBuffer, RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, bool ProcessAllSections, - std::function<Error(std::unique_ptr<LoadedObjectInfo>, - std::map<StringRef, JITEvaluatedSymbol>)> + unique_function<Error(std::unique_ptr<LoadedObjectInfo>, + std::map<StringRef, JITEvaluatedSymbol>)> OnLoaded, - std::function<void(Error)> OnEmitted); + unique_function<void(Error)> OnEmitted); // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public // interface. @@ -291,14 +292,14 @@ private: // but ORC's RTDyldObjectLinkingLayer2. Internally it constructs a RuntimeDyld // instance and uses continuation passing to perform the fix-up and finalize // steps asynchronously. -void jitLinkForORC(object::ObjectFile &Obj, - std::unique_ptr<MemoryBuffer> UnderlyingBuffer, - RuntimeDyld::MemoryManager &MemMgr, - JITSymbolResolver &Resolver, bool ProcessAllSections, - std::function<Error(std::unique_ptr<LoadedObjectInfo>, - std::map<StringRef, JITEvaluatedSymbol>)> - OnLoaded, - std::function<void(Error)> OnEmitted); +void jitLinkForORC( + object::ObjectFile &Obj, std::unique_ptr<MemoryBuffer> UnderlyingBuffer, + RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, + bool ProcessAllSections, + unique_function<Error(std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, + std::map<StringRef, JITEvaluatedSymbol>)> + OnLoaded, + unique_function<void(Error)> OnEmitted); } // end namespace llvm diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h index 06cc09e1cfc7..e6b280465f72 100644 --- a/include/llvm/IR/Attributes.h +++ b/include/llvm/IR/Attributes.h @@ -22,6 +22,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/llvm-config.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include <bitset> #include <cassert> @@ -94,8 +95,8 @@ public: /// Return a uniquified Attribute object that has the specific /// alignment set. - static Attribute getWithAlignment(LLVMContext &Context, uint64_t Align); - static Attribute getWithStackAlignment(LLVMContext &Context, uint64_t Align); + static Attribute getWithAlignment(LLVMContext &Context, Align Alignment); + static Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment); static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes); static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, @@ -150,11 +151,11 @@ public: /// Returns the alignment field of an attribute as a byte alignment /// value. - unsigned getAlignment() const; + MaybeAlign getAlignment() const; /// Returns the stack alignment field of an attribute as a byte /// alignment value. - unsigned getStackAlignment() const; + MaybeAlign getStackAlignment() const; /// Returns the number of dereferenceable bytes from the /// dereferenceable attribute. @@ -284,8 +285,8 @@ public: /// Return the target-dependent attribute object. Attribute getAttribute(StringRef Kind) const; - unsigned getAlignment() const; - unsigned getStackAlignment() const; + MaybeAlign getAlignment() const; + MaybeAlign getStackAlignment() const; uint64_t getDereferenceableBytes() const; uint64_t getDereferenceableOrNullBytes() const; Type *getByValType() const; @@ -603,16 +604,16 @@ public: } /// Return the alignment of the return value. - unsigned getRetAlignment() const; + MaybeAlign getRetAlignment() const; /// Return the alignment for the specified function parameter. - unsigned getParamAlignment(unsigned ArgNo) const; + MaybeAlign getParamAlignment(unsigned ArgNo) const; /// Return the byval type for the specified function parameter. Type *getParamByValType(unsigned ArgNo) const; /// Get the stack alignment. - unsigned getStackAlignment(unsigned Index) const; + MaybeAlign getStackAlignment(unsigned Index) const; /// Get the number of dereferenceable bytes (or zero if unknown). uint64_t getDereferenceableBytes(unsigned Index) const; @@ -704,9 +705,9 @@ template <> struct DenseMapInfo<AttributeList> { /// equality, presence of attributes, etc. class AttrBuilder { std::bitset<Attribute::EndAttrKinds> Attrs; - std::map<std::string, std::string> TargetDepAttrs; - uint64_t Alignment = 0; - uint64_t StackAlignment = 0; + std::map<std::string, std::string, std::less<>> TargetDepAttrs; + MaybeAlign Alignment; + MaybeAlign StackAlignment; uint64_t DerefBytes = 0; uint64_t DerefOrNullBytes = 0; uint64_t AllocSizeArgs = 0; @@ -773,10 +774,10 @@ public: bool hasAlignmentAttr() const; /// Retrieve the alignment attribute, if it exists. - uint64_t getAlignment() const { return Alignment; } + MaybeAlign getAlignment() const { return Alignment; } /// Retrieve the stack alignment attribute, if it exists. - uint64_t getStackAlignment() const { return StackAlignment; } + MaybeAlign getStackAlignment() const { return StackAlignment; } /// Retrieve the number of dereferenceable bytes, if the /// dereferenceable attribute exists (zero is returned otherwise). @@ -793,13 +794,29 @@ public: /// doesn't exist, pair(0, 0) is returned. std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; + /// This turns an alignment into the form used internally in Attribute. + /// This call has no effect if Align is not set. + AttrBuilder &addAlignmentAttr(MaybeAlign Align); + /// This turns an int alignment (which must be a power of 2) into the /// form used internally in Attribute. - AttrBuilder &addAlignmentAttr(unsigned Align); + /// This call has no effect if Align is 0. + /// Deprecated, use the version using a MaybeAlign. + inline AttrBuilder &addAlignmentAttr(unsigned Align) { + return addAlignmentAttr(MaybeAlign(Align)); + } + + /// This turns a stack alignment into the form used internally in Attribute. + /// This call has no effect if Align is not set. + AttrBuilder &addStackAlignmentAttr(MaybeAlign Align); /// This turns an int stack alignment (which must be a power of 2) into /// the form used internally in Attribute. - AttrBuilder &addStackAlignmentAttr(unsigned Align); + /// This call has no effect if Align is 0. + /// Deprecated, use the version using a MaybeAlign. + inline AttrBuilder &addStackAlignmentAttr(unsigned Align) { + return addStackAlignmentAttr(MaybeAlign(Align)); + } /// This turns the number of dereferenceable bytes into the form used /// internally in Attribute. diff --git a/include/llvm/IR/AutoUpgrade.h b/include/llvm/IR/AutoUpgrade.h index 017ad93d8a2a..66f38e5b55d1 100644 --- a/include/llvm/IR/AutoUpgrade.h +++ b/include/llvm/IR/AutoUpgrade.h @@ -54,9 +54,9 @@ namespace llvm { /// module is modified. bool UpgradeModuleFlags(Module &M); - /// This checks for objc retain release marker which should be upgraded. It - /// returns true if module is modified. - bool UpgradeRetainReleaseMarker(Module &M); + /// Convert calls to ARC runtime functions to intrinsic calls and upgrade the + /// old retain release marker to new module flag format. + void UpgradeARCRuntime(Module &M); void UpgradeSectionAttributes(Module &M); @@ -87,6 +87,10 @@ namespace llvm { /// Upgrade the loop attachment metadata node. MDNode *upgradeInstructionLoopAttachment(MDNode &N); + /// Upgrade the datalayout string by adding a section for address space + /// pointers. + std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple); + } // End llvm namespace #endif diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h index 69555af50e1f..d594145f8636 100644 --- a/include/llvm/IR/BasicBlock.h +++ b/include/llvm/IR/BasicBlock.h @@ -192,6 +192,11 @@ public: std::function<bool(Instruction &)>>> instructionsWithoutDebug(); + /// Return the size of the basic block ignoring debug instructions + filter_iterator<BasicBlock::const_iterator, + std::function<bool(const Instruction &)>>::difference_type + sizeWithoutDebug() const; + /// Unlink 'this' from the containing function, but do not delete it. void removeFromParent(); diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h index b47a96c5d5fa..13b1ae8d0e32 100644 --- a/include/llvm/IR/CallSite.h +++ b/include/llvm/IR/CallSite.h @@ -854,6 +854,15 @@ public: return CI.ParameterEncoding[0]; } + /// Return the use of the callee value in the underlying instruction. Only + /// valid for callback calls! + const Use &getCalleeUseForCallback() const { + int CalleeArgIdx = getCallArgOperandNoForCallee(); + assert(CalleeArgIdx >= 0 && + unsigned(CalleeArgIdx) < getInstruction()->getNumOperands()); + return getInstruction()->getOperandUse(CalleeArgIdx); + } + /// Return the pointer to function that is being called. Value *getCalledValue() const { if (isDirectCall()) diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h index 399c6ad521fa..c1c979c2e2ab 100644 --- a/include/llvm/IR/CallingConv.h +++ b/include/llvm/IR/CallingConv.h @@ -75,6 +75,11 @@ namespace CallingConv { // CXX_FAST_TLS - Calling convention for access functions. CXX_FAST_TLS = 17, + /// Tail - This calling convention attemps to make calls as fast as + /// possible while guaranteeing that tail call optimization can always + /// be performed. + Tail = 18, + // Target - This is the start of the target-specific calling conventions, // e.g. fastcall and thiscall on X86. FirstTargetCC = 64, @@ -222,6 +227,14 @@ namespace CallingConv { // Calling convention between AArch64 Advanced SIMD functions AArch64_VectorCall = 97, + /// Calling convention between AArch64 SVE functions + AArch64_SVE_VectorCall = 98, + + /// Calling convention for emscripten __invoke_* functions. The first + /// argument is required to be the function ptr being indirectly called. + /// The remainder matches the regular calling convention. + WASM_EmscriptenInvoke = 99, + /// The highest possible calling convention ID. Must be some 2^k - 1. MaxID = 1023 }; diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h index 931576651224..2b6a6e4141b9 100644 --- a/include/llvm/IR/Constant.h +++ b/include/llvm/IR/Constant.h @@ -86,6 +86,12 @@ public: /// floating-point constant with all NaN elements. bool isNaN() const; + /// Return true if this constant and a constant 'Y' are element-wise equal. + /// This is identical to just comparing the pointers, with the exception that + /// for vectors, if only one of the constants has an `undef` element in some + /// lane, the constants still match. + bool isElementWiseEqual(Value *Y) const; + /// Return true if this is a vector constant that includes any undefined /// elements. bool containsUndefElement() const; diff --git a/include/llvm/IR/ConstantRange.h b/include/llvm/IR/ConstantRange.h index 91f3f31abe17..964f9e8e9bc9 100644 --- a/include/llvm/IR/ConstantRange.h +++ b/include/llvm/IR/ConstantRange.h @@ -330,9 +330,13 @@ public: /// from an addition of a value in this range and a value in \p Other. ConstantRange add(const ConstantRange &Other) const; - /// Return a new range representing the possible values resulting from a - /// known NSW addition of a value in this range and \p Other constant. - ConstantRange addWithNoSignedWrap(const APInt &Other) const; + /// Return a new range representing the possible values resulting + /// from an addition with wrap type \p NoWrapKind of a value in this + /// range and a value in \p Other. + /// If the result range is disjoint, the preferred range is determined by the + /// \p PreferredRangeType. + ConstantRange addWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind, + PreferredRangeType RangeType = Smallest) const; /// Return a new range representing the possible values resulting /// from a subtraction of a value in this range and a value in \p Other. diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h index ac9770a15120..85093dd218f8 100644 --- a/include/llvm/IR/DataLayout.h +++ b/include/llvm/IR/DataLayout.h @@ -25,10 +25,11 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Type.h" -#include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/TypeSize.h" #include <cassert> #include <cstdint> #include <string> @@ -71,11 +72,11 @@ struct LayoutAlignElem { /// Alignment type from \c AlignTypeEnum unsigned AlignType : 8; unsigned TypeBitWidth : 24; - unsigned ABIAlign : 16; - unsigned PrefAlign : 16; + Align ABIAlign; + Align PrefAlign; - static LayoutAlignElem get(AlignTypeEnum align_type, unsigned abi_align, - unsigned pref_align, uint32_t bit_width); + static LayoutAlignElem get(AlignTypeEnum align_type, Align abi_align, + Align pref_align, uint32_t bit_width); bool operator==(const LayoutAlignElem &rhs) const; }; @@ -87,15 +88,15 @@ struct LayoutAlignElem { /// \note The unusual order of elements in the structure attempts to reduce /// padding and make the structure slightly more cache friendly. struct PointerAlignElem { - unsigned ABIAlign; - unsigned PrefAlign; + Align ABIAlign; + Align PrefAlign; uint32_t TypeByteWidth; uint32_t AddressSpace; uint32_t IndexWidth; /// Initializer - static PointerAlignElem get(uint32_t AddressSpace, unsigned ABIAlign, - unsigned PrefAlign, uint32_t TypeByteWidth, + static PointerAlignElem get(uint32_t AddressSpace, Align ABIAlign, + Align PrefAlign, uint32_t TypeByteWidth, uint32_t IndexWidth); bool operator==(const PointerAlignElem &rhs) const; @@ -120,10 +121,10 @@ private: bool BigEndian; unsigned AllocaAddrSpace; - unsigned StackNaturalAlign; + MaybeAlign StackNaturalAlign; unsigned ProgramAddrSpace; - unsigned FunctionPtrAlign; + MaybeAlign FunctionPtrAlign; FunctionPtrAlignType TheFunctionPtrAlignType; enum ManglingModeT { @@ -172,16 +173,15 @@ private: /// well-defined bitwise representation. SmallVector<unsigned, 8> NonIntegralAddressSpaces; - void setAlignment(AlignTypeEnum align_type, unsigned abi_align, - unsigned pref_align, uint32_t bit_width); - unsigned getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width, - bool ABIAlign, Type *Ty) const; - void setPointerAlignment(uint32_t AddrSpace, unsigned ABIAlign, - unsigned PrefAlign, uint32_t TypeByteWidth, - uint32_t IndexWidth); + void setAlignment(AlignTypeEnum align_type, Align abi_align, Align pref_align, + uint32_t bit_width); + Align getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width, + bool ABIAlign, Type *Ty) const; + void setPointerAlignment(uint32_t AddrSpace, Align ABIAlign, Align PrefAlign, + uint32_t TypeByteWidth, uint32_t IndexWidth); /// Internal helper method that returns requested alignment for type. - unsigned getAlignment(Type *Ty, bool abi_or_pref) const; + Align getAlignment(Type *Ty, bool abi_or_pref) const; /// Parses a target data specification string. Assert if the string is /// malformed. @@ -261,17 +261,21 @@ public: bool isIllegalInteger(uint64_t Width) const { return !isLegalInteger(Width); } /// Returns true if the given alignment exceeds the natural stack alignment. - bool exceedsNaturalStackAlignment(unsigned Align) const { - return (StackNaturalAlign != 0) && (Align > StackNaturalAlign); + bool exceedsNaturalStackAlignment(Align Alignment) const { + return StackNaturalAlign && (Alignment > StackNaturalAlign); + } + + Align getStackAlignment() const { + assert(StackNaturalAlign && "StackNaturalAlign must be defined"); + return *StackNaturalAlign; } - unsigned getStackAlignment() const { return StackNaturalAlign; } unsigned getAllocaAddrSpace() const { return AllocaAddrSpace; } /// Returns the alignment of function pointers, which may or may not be /// related to the alignment of functions. /// \see getFunctionPtrAlignType - unsigned getFunctionPtrAlign() const { return FunctionPtrAlign; } + MaybeAlign getFunctionPtrAlign() const { return FunctionPtrAlign; } /// Return the type of function pointer alignment. /// \see getFunctionPtrAlign @@ -344,12 +348,12 @@ public: } /// Layout pointer alignment - unsigned getPointerABIAlignment(unsigned AS) const; + Align getPointerABIAlignment(unsigned AS) const; /// Return target's alignment for stack-based pointers /// FIXME: The defaults need to be removed once all of /// the backends/clients are updated. - unsigned getPointerPrefAlignment(unsigned AS = 0) const; + Align getPointerPrefAlignment(unsigned AS = 0) const; /// Layout pointer size /// FIXME: The defaults need to be removed once all of @@ -433,23 +437,33 @@ public: /// Returns the number of bits necessary to hold the specified type. /// + /// If Ty is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// For example, returns 36 for i36 and 80 for x86_fp80. The type passed must /// have a size (Type::isSized() must return true). - uint64_t getTypeSizeInBits(Type *Ty) const; + TypeSize getTypeSizeInBits(Type *Ty) const; /// Returns the maximum number of bytes that may be overwritten by /// storing the specified type. /// + /// If Ty is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// For example, returns 5 for i36 and 10 for x86_fp80. - uint64_t getTypeStoreSize(Type *Ty) const { - return (getTypeSizeInBits(Ty) + 7) / 8; + TypeSize getTypeStoreSize(Type *Ty) const { + TypeSize BaseSize = getTypeSizeInBits(Ty); + return { (BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable() }; } /// Returns the maximum number of bits that may be overwritten by /// storing the specified type; always a multiple of 8. /// + /// If Ty is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// For example, returns 40 for i36 and 80 for x86_fp80. - uint64_t getTypeStoreSizeInBits(Type *Ty) const { + TypeSize getTypeStoreSizeInBits(Type *Ty) const { return 8 * getTypeStoreSize(Ty); } @@ -464,9 +478,12 @@ public: /// Returns the offset in bytes between successive objects of the /// specified type, including alignment padding. /// + /// If Ty is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// This is the amount that alloca reserves for this type. For example, /// returns 12 or 16 for x86_fp80, depending on alignment. - uint64_t getTypeAllocSize(Type *Ty) const { + TypeSize getTypeAllocSize(Type *Ty) const { // Round up to the next alignment boundary. return alignTo(getTypeStoreSize(Ty), getABITypeAlignment(Ty)); } @@ -474,18 +491,28 @@ public: /// Returns the offset in bits between successive objects of the /// specified type, including alignment padding; always a multiple of 8. /// + /// If Ty is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// This is the amount that alloca reserves for this type. For example, /// returns 96 or 128 for x86_fp80, depending on alignment. - uint64_t getTypeAllocSizeInBits(Type *Ty) const { + TypeSize getTypeAllocSizeInBits(Type *Ty) const { return 8 * getTypeAllocSize(Ty); } /// Returns the minimum ABI-required alignment for the specified type. unsigned getABITypeAlignment(Type *Ty) const; + /// Helper function to return `Alignment` if it's set or the result of + /// `getABITypeAlignment(Ty)`, in any case the result is a valid alignment. + inline Align getValueOrABITypeAlignment(MaybeAlign Alignment, + Type *Ty) const { + return Alignment ? *Alignment : Align(getABITypeAlignment(Ty)); + } + /// Returns the minimum ABI-required alignment for an integer type of /// the specified bitwidth. - unsigned getABIIntegerTypeAlignment(unsigned BitWidth) const; + Align getABIIntegerTypeAlignment(unsigned BitWidth) const; /// Returns the preferred stack/global alignment for the specified /// type. @@ -493,10 +520,6 @@ public: /// This is always at least as good as the ABI alignment. unsigned getPrefTypeAlignment(Type *Ty) const; - /// Returns the preferred alignment for the specified type, returned as - /// log2 of the value (a shift amount). - unsigned getPreferredTypeAlignmentShift(Type *Ty) const; - /// Returns an integer type with size at least as big as that of a /// pointer in the given address space. IntegerType *getIntPtrType(LLVMContext &C, unsigned AddressSpace = 0) const; @@ -561,7 +584,7 @@ inline LLVMTargetDataRef wrap(const DataLayout *P) { /// based on the DataLayout structure. class StructLayout { uint64_t StructSize; - unsigned StructAlignment; + Align StructAlignment; unsigned IsPadded : 1; unsigned NumElements : 31; uint64_t MemberOffsets[1]; // variable sized array! @@ -571,7 +594,7 @@ public: uint64_t getSizeInBits() const { return 8 * StructSize; } - unsigned getAlignment() const { return StructAlignment; } + Align getAlignment() const { return StructAlignment; } /// Returns whether the struct has padding or not between its fields. /// NB: Padding in nested element is not taken into account. @@ -598,13 +621,13 @@ private: // The implementation of this method is provided inline as it is particularly // well suited to constant folding when called on a specific Type subclass. -inline uint64_t DataLayout::getTypeSizeInBits(Type *Ty) const { +inline TypeSize DataLayout::getTypeSizeInBits(Type *Ty) const { assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!"); switch (Ty->getTypeID()) { case Type::LabelTyID: - return getPointerSizeInBits(0); + return TypeSize::Fixed(getPointerSizeInBits(0)); case Type::PointerTyID: - return getPointerSizeInBits(Ty->getPointerAddressSpace()); + return TypeSize::Fixed(getPointerSizeInBits(Ty->getPointerAddressSpace())); case Type::ArrayTyID: { ArrayType *ATy = cast<ArrayType>(Ty); return ATy->getNumElements() * @@ -612,26 +635,30 @@ inline uint64_t DataLayout::getTypeSizeInBits(Type *Ty) const { } case Type::StructTyID: // Get the layout annotation... which is lazily created on demand. - return getStructLayout(cast<StructType>(Ty))->getSizeInBits(); + return TypeSize::Fixed( + getStructLayout(cast<StructType>(Ty))->getSizeInBits()); case Type::IntegerTyID: - return Ty->getIntegerBitWidth(); + return TypeSize::Fixed(Ty->getIntegerBitWidth()); case Type::HalfTyID: - return 16; + return TypeSize::Fixed(16); case Type::FloatTyID: - return 32; + return TypeSize::Fixed(32); case Type::DoubleTyID: case Type::X86_MMXTyID: - return 64; + return TypeSize::Fixed(64); case Type::PPC_FP128TyID: case Type::FP128TyID: - return 128; + return TypeSize::Fixed(128); // In memory objects this is always aligned to a higher boundary, but // only 80 bits contain information. case Type::X86_FP80TyID: - return 80; + return TypeSize::Fixed(80); case Type::VectorTyID: { VectorType *VTy = cast<VectorType>(Ty); - return VTy->getNumElements() * getTypeSizeInBits(VTy->getElementType()); + auto EltCnt = VTy->getElementCount(); + uint64_t MinBits = EltCnt.Min * + getTypeSizeInBits(VTy->getElementType()).getFixedSize(); + return TypeSize(MinBits, EltCnt.Scalable); } default: llvm_unreachable("DataLayout::getTypeSizeInBits(): Unsupported type"); diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def index 07e3d6bdc9e5..f90c580f10ef 100644 --- a/include/llvm/IR/DebugInfoFlags.def +++ b/include/llvm/IR/DebugInfoFlags.def @@ -31,7 +31,8 @@ HANDLE_DI_FLAG(2, Protected) HANDLE_DI_FLAG(3, Public) HANDLE_DI_FLAG((1 << 2), FwdDecl) HANDLE_DI_FLAG((1 << 3), AppleBlock) -HANDLE_DI_FLAG((1 << 4), BlockByrefStruct) +// Used to be BlockByRef, can be reused for anything except DICompositeType. +HANDLE_DI_FLAG((1 << 4), ReservedBit4) HANDLE_DI_FLAG((1 << 5), Virtual) HANDLE_DI_FLAG((1 << 6), Artificial) HANDLE_DI_FLAG((1 << 7), Explicit) @@ -42,8 +43,7 @@ HANDLE_DI_FLAG((1 << 11), Vector) HANDLE_DI_FLAG((1 << 12), StaticMember) HANDLE_DI_FLAG((1 << 13), LValueReference) HANDLE_DI_FLAG((1 << 14), RValueReference) -// 15 was formerly ExternalTypeRef, but this was never used. -HANDLE_DI_FLAG((1 << 15), Reserved) +HANDLE_DI_FLAG((1 << 15), ExportSymbols) HANDLE_DI_FLAG((1 << 16), SingleInheritance) HANDLE_DI_FLAG((2 << 16), MultipleInheritance) HANDLE_DI_FLAG((3 << 16), VirtualInheritance) diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h index 9dc6dfbb0f68..28a59576b7c6 100644 --- a/include/llvm/IR/DebugInfoMetadata.h +++ b/include/llvm/IR/DebugInfoMetadata.h @@ -650,7 +650,6 @@ public: } bool isForwardDecl() const { return getFlags() & FlagFwdDecl; } bool isAppleBlockExtension() const { return getFlags() & FlagAppleBlock; } - bool isBlockByrefStruct() const { return getFlags() & FlagBlockByrefStruct; } bool isVirtual() const { return getFlags() & FlagVirtual; } bool isArtificial() const { return getFlags() & FlagArtificial; } bool isObjectPointer() const { return getFlags() & FlagObjectPointer; } @@ -668,6 +667,7 @@ public: } bool isBigEndian() const { return getFlags() & FlagBigEndian; } bool isLittleEndian() const { return getFlags() & FlagLittleEndian; } + bool getExportSymbols() const { return getFlags() & FlagExportSymbols; } static bool classof(const Metadata *MD) { switch (MD->getMetadataID()) { @@ -2569,7 +2569,7 @@ public: /// (This is the only configuration of entry values that is supported.) bool isEntryValue() const { return getNumElements() > 0 && - getElement(0) == dwarf::DW_OP_entry_value; + getElement(0) == dwarf::DW_OP_LLVM_entry_value; } }; diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h index 3c1d4278905f..20097ef3f31a 100644 --- a/include/llvm/IR/DerivedTypes.h +++ b/include/llvm/IR/DerivedTypes.h @@ -23,7 +23,7 @@ #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/ScalableSize.h" +#include "llvm/Support/TypeSize.h" #include <cassert> #include <cstdint> @@ -62,6 +62,11 @@ public: /// Get or create an IntegerType instance. static IntegerType *get(LLVMContext &C, unsigned NumBits); + /// Returns type twice as wide the input type. + IntegerType *getExtendedType() const { + return Type::getIntNTy(getContext(), 2 * getScalarSizeInBits()); + } + /// Get the number of bits in this IntegerType unsigned getBitWidth() const { return getSubclassData(); } @@ -470,21 +475,47 @@ public: /// This static method is like getInteger except that the element types are /// twice as wide as the elements in the input type. static VectorType *getExtendedElementVectorType(VectorType *VTy) { - unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); - Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2); - return VectorType::get(EltTy, VTy->getElementCount()); + assert(VTy->isIntOrIntVectorTy() && "VTy expected to be a vector of ints."); + auto *EltTy = cast<IntegerType>(VTy->getElementType()); + return VectorType::get(EltTy->getExtendedType(), VTy->getElementCount()); } - /// This static method is like getInteger except that the element types are - /// half as wide as the elements in the input type. + // This static method gets a VectorType with the same number of elements as + // the input type, and the element type is an integer or float type which + // is half as wide as the elements in the input type. static VectorType *getTruncatedElementVectorType(VectorType *VTy) { - unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); - assert((EltBits & 1) == 0 && - "Cannot truncate vector element with odd bit-width"); - Type *EltTy = IntegerType::get(VTy->getContext(), EltBits / 2); + Type *EltTy; + if (VTy->getElementType()->isFloatingPointTy()) { + switch(VTy->getElementType()->getTypeID()) { + case DoubleTyID: + EltTy = Type::getFloatTy(VTy->getContext()); + break; + case FloatTyID: + EltTy = Type::getHalfTy(VTy->getContext()); + break; + default: + llvm_unreachable("Cannot create narrower fp vector element type"); + } + } else { + unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); + assert((EltBits & 1) == 0 && + "Cannot truncate vector element with odd bit-width"); + EltTy = IntegerType::get(VTy->getContext(), EltBits / 2); + } return VectorType::get(EltTy, VTy->getElementCount()); } + // This static method returns a VectorType with a smaller number of elements + // of a larger type than the input element type. For example, a <16 x i8> + // subdivided twice would return <4 x i32> + static VectorType *getSubdividedVectorType(VectorType *VTy, int NumSubdivs) { + for (int i = 0; i < NumSubdivs; ++i) { + VTy = VectorType::getDoubleElementsVectorType(VTy); + VTy = VectorType::getTruncatedElementVectorType(VTy); + } + return VTy; + } + /// This static method returns a VectorType with half as many elements as the /// input type and the same element type. static VectorType *getHalfElementsVectorType(VectorType *VTy) { @@ -540,6 +571,10 @@ bool Type::getVectorIsScalable() const { return cast<VectorType>(this)->isScalable(); } +ElementCount Type::getVectorElementCount() const { + return cast<VectorType>(this)->getElementCount(); +} + /// Class to represent pointers. class PointerType : public Type { explicit PointerType(Type *ElType, unsigned AddrSpace); @@ -577,6 +612,26 @@ public: } }; +Type *Type::getExtendedType() const { + assert( + isIntOrIntVectorTy() && + "Original type expected to be a vector of integers or a scalar integer."); + if (auto *VTy = dyn_cast<VectorType>(this)) + return VectorType::getExtendedElementVectorType( + const_cast<VectorType *>(VTy)); + return cast<IntegerType>(this)->getExtendedType(); +} + +Type *Type::getWithNewBitWidth(unsigned NewBitWidth) const { + assert( + isIntOrIntVectorTy() && + "Original type expected to be a vector of integers or a scalar integer."); + Type *NewType = getIntNTy(getContext(), NewBitWidth); + if (isVectorTy()) + NewType = VectorType::get(NewType, getVectorElementCount()); + return NewType; +} + unsigned Type::getPointerAddressSpace() const { return cast<PointerType>(getScalarType())->getAddressSpace(); } diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h index 373663289dbd..ec469982d378 100644 --- a/include/llvm/IR/DiagnosticInfo.h +++ b/include/llvm/IR/DiagnosticInfo.h @@ -74,8 +74,10 @@ enum DiagnosticKind { DK_LastMachineRemark = DK_MachineOptimizationRemarkAnalysis, DK_MIRParser, DK_PGOProfile, + DK_MisExpect, DK_Unsupported, - DK_FirstPluginKind + DK_FirstPluginKind // Must be last value to work with + // getNextAvailablePluginDiagnosticKind }; /// Get the next available kind ID for a plugin diagnostic. @@ -663,7 +665,7 @@ public: private: /// The IR value (currently basic block) that the optimization operates on. /// This is currently used to provide run-time hotness information with PGO. - const Value *CodeRegion; + const Value *CodeRegion = nullptr; }; /// Diagnostic information for applied optimization remarks. @@ -1002,6 +1004,25 @@ public: void print(DiagnosticPrinter &DP) const override; }; +/// Diagnostic information for MisExpect analysis. +class DiagnosticInfoMisExpect : public DiagnosticInfoWithLocationBase { +public: + DiagnosticInfoMisExpect(const Instruction *Inst, Twine &Msg); + + /// \see DiagnosticInfo::print. + void print(DiagnosticPrinter &DP) const override; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_MisExpect; + } + + const Twine &getMsg() const { return Msg; } + +private: + /// Message to report. + const Twine &Msg; +}; + } // end namespace llvm #endif // LLVM_IR_DIAGNOSTICINFO_H diff --git a/include/llvm/IR/FixedMetadataKinds.def b/include/llvm/IR/FixedMetadataKinds.def new file mode 100644 index 000000000000..0e1ffef58672 --- /dev/null +++ b/include/llvm/IR/FixedMetadataKinds.def @@ -0,0 +1,43 @@ +/*===-- FixedMetadataKinds.def - Fixed metadata kind IDs -------*- C++ -*-=== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_FIXED_MD_KIND +#error "LLVM_FIXED_MD_KIND(EnumID, Name, Value) is not defined." +#endif + +LLVM_FIXED_MD_KIND(MD_dbg, "dbg", 0) +LLVM_FIXED_MD_KIND(MD_tbaa, "tbaa", 1) +LLVM_FIXED_MD_KIND(MD_prof, "prof", 2) +LLVM_FIXED_MD_KIND(MD_fpmath, "fpmath", 3) +LLVM_FIXED_MD_KIND(MD_range, "range", 4) +LLVM_FIXED_MD_KIND(MD_tbaa_struct, "tbaa.struct", 5) +LLVM_FIXED_MD_KIND(MD_invariant_load, "invariant.load", 6) +LLVM_FIXED_MD_KIND(MD_alias_scope, "alias.scope", 7) +LLVM_FIXED_MD_KIND(MD_noalias, "noalias", 8) +LLVM_FIXED_MD_KIND(MD_nontemporal, "nontemporal", 9) +LLVM_FIXED_MD_KIND(MD_mem_parallel_loop_access, + "llvm.mem.parallel_loop_access", 10) +LLVM_FIXED_MD_KIND(MD_nonnull, "nonnull", 11) +LLVM_FIXED_MD_KIND(MD_dereferenceable, "dereferenceable", 12) +LLVM_FIXED_MD_KIND(MD_dereferenceable_or_null, "dereferenceable_or_null", 13) +LLVM_FIXED_MD_KIND(MD_make_implicit, "make.implicit", 14) +LLVM_FIXED_MD_KIND(MD_unpredictable, "unpredictable", 15) +LLVM_FIXED_MD_KIND(MD_invariant_group, "invariant.group", 16) +LLVM_FIXED_MD_KIND(MD_align, "align", 17) +LLVM_FIXED_MD_KIND(MD_loop, "llvm.loop", 18) +LLVM_FIXED_MD_KIND(MD_type, "type", 19) +LLVM_FIXED_MD_KIND(MD_section_prefix, "section_prefix", 20) +LLVM_FIXED_MD_KIND(MD_absolute_symbol, "absolute_symbol", 21) +LLVM_FIXED_MD_KIND(MD_associated, "associated", 22) +LLVM_FIXED_MD_KIND(MD_callees, "callees", 23) +LLVM_FIXED_MD_KIND(MD_irr_loop, "irr_loop", 24) +LLVM_FIXED_MD_KIND(MD_access_group, "llvm.access.group", 25) +LLVM_FIXED_MD_KIND(MD_callback, "callback", 26) +LLVM_FIXED_MD_KIND(MD_preserve_access_index, "llvm.preserve.access.index", 27) +LLVM_FIXED_MD_KIND(MD_misexpect, "misexpect", 28) +LLVM_FIXED_MD_KIND(MD_vcall_visibility, "vcall_visibility", 29) diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h index 7fa61e12f431..d586a9460d2b 100644 --- a/include/llvm/IR/Function.h +++ b/include/llvm/IR/Function.h @@ -343,7 +343,10 @@ public: unsigned getFnStackAlignment() const { if (!hasFnAttribute(Attribute::StackAlignment)) return 0; - return AttributeSets.getStackAlignment(AttributeList::FunctionIndex); + if (const auto MA = + AttributeSets.getStackAlignment(AttributeList::FunctionIndex)) + return MA->value(); + return 0; } /// hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm @@ -433,7 +436,9 @@ public: /// Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned ArgNo) const { - return AttributeSets.getParamAlignment(ArgNo); + if (const auto MA = AttributeSets.getParamAlignment(ArgNo)) + return MA->value(); + return 0; } /// Extract the byval type for a parameter. @@ -710,6 +715,12 @@ public: return Arguments + NumArgs; } + Argument* getArg(unsigned i) const { + assert (i < NumArgs && "getArg() out of range!"); + CheckLazyArguments(); + return Arguments + i; + } + iterator_range<arg_iterator> args() { return make_range(arg_begin(), arg_end()); } diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h index 3cd405701300..f2d9b9676ec9 100644 --- a/include/llvm/IR/GlobalAlias.h +++ b/include/llvm/IR/GlobalAlias.h @@ -58,10 +58,6 @@ public: // Linkage, Type, Parent and AddressSpace taken from the Aliasee. static GlobalAlias *create(const Twine &Name, GlobalValue *Aliasee); - void copyAttributesFrom(const GlobalValue *Src) { - GlobalValue::copyAttributesFrom(Src); - } - /// removeFromParent - This method unlinks 'this' from the containing module, /// but does not delete it. /// diff --git a/include/llvm/IR/GlobalIFunc.h b/include/llvm/IR/GlobalIFunc.h index bc0d3c053cce..0fdae917878a 100644 --- a/include/llvm/IR/GlobalIFunc.h +++ b/include/llvm/IR/GlobalIFunc.h @@ -46,10 +46,6 @@ public: LinkageTypes Linkage, const Twine &Name, Constant *Resolver, Module *Parent); - void copyAttributesFrom(const GlobalIFunc *Src) { - GlobalValue::copyAttributesFrom(Src); - } - /// This method unlinks 'this' from the containing module, but does not /// delete it. void removeFromParent(); diff --git a/include/llvm/IR/GlobalIndirectSymbol.h b/include/llvm/IR/GlobalIndirectSymbol.h index 8bc3f90b94aa..d996237aa3ef 100644 --- a/include/llvm/IR/GlobalIndirectSymbol.h +++ b/include/llvm/IR/GlobalIndirectSymbol.h @@ -42,6 +42,10 @@ public: /// Provide fast operand accessors DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); + void copyAttributesFrom(const GlobalValue *Src) { + GlobalValue::copyAttributesFrom(Src); + } + /// These methods set and retrieve indirect symbol. void setIndirectSymbol(Constant *Symbol) { setOperand(0, Symbol); @@ -54,9 +58,7 @@ public: static_cast<const GlobalIndirectSymbol *>(this)->getIndirectSymbol()); } - const GlobalObject *getBaseObject() const { - return dyn_cast<GlobalObject>(getIndirectSymbol()->stripInBoundsOffsets()); - } + const GlobalObject *getBaseObject() const; GlobalObject *getBaseObject() { return const_cast<GlobalObject *>( static_cast<const GlobalIndirectSymbol *>(this)->getBaseObject()); diff --git a/include/llvm/IR/GlobalObject.h b/include/llvm/IR/GlobalObject.h index b8ab6140ebe7..ce81eb9f0719 100644 --- a/include/llvm/IR/GlobalObject.h +++ b/include/llvm/IR/GlobalObject.h @@ -17,6 +17,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Value.h" +#include "llvm/Support/Alignment.h" #include <string> #include <utility> @@ -27,6 +28,20 @@ class MDNode; class Metadata; class GlobalObject : public GlobalValue { +public: + // VCallVisibility - values for visibility metadata attached to vtables. This + // describes the scope in which a virtual call could end up being dispatched + // through this vtable. + enum VCallVisibility { + // Type is potentially visible to external code. + VCallVisibilityPublic = 0, + // Type is only visible to code which will be in the current Module after + // LTO internalization. + VCallVisibilityLinkageUnit = 1, + // Type is only visible to code in the current Module. + VCallVisibilityTranslationUnit = 2, + }; + protected: GlobalObject(Type *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, LinkageTypes Linkage, const Twine &Name, @@ -58,9 +73,14 @@ public: unsigned getAlignment() const { unsigned Data = getGlobalValueSubClassData(); unsigned AlignmentData = Data & AlignmentMask; - return (1u << AlignmentData) >> 1; + MaybeAlign Align = decodeMaybeAlign(AlignmentData); + return Align ? Align->value() : 0; } - void setAlignment(unsigned Align); + + /// FIXME: Remove this setter once the migration to MaybeAlign is over. + LLVM_ATTRIBUTE_DEPRECATED(void setAlignment(unsigned Align), + "Please use `void setAlignment(MaybeAlign Align)`"); + void setAlignment(MaybeAlign Align); unsigned getGlobalObjectSubClassData() const { unsigned ValueData = getGlobalValueSubClassData(); @@ -158,6 +178,8 @@ public: void copyMetadata(const GlobalObject *Src, unsigned Offset); void addTypeMetadata(unsigned Offset, Metadata *TypeID); + void addVCallVisibilityMetadata(VCallVisibility Visibility); + VCallVisibility getVCallVisibility() const; protected: void copyAttributesFrom(const GlobalObject *Src); diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h index 2e2c8c477913..2c730bc312e4 100644 --- a/include/llvm/IR/GlobalVariable.h +++ b/include/llvm/IR/GlobalVariable.h @@ -243,6 +243,7 @@ public: bool hasImplicitSection() const { return getAttributes().hasAttribute("bss-section") || getAttributes().hasAttribute("data-section") || + getAttributes().hasAttribute("relro-section") || getAttributes().hasAttribute("rodata-section"); } diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index a74364dffb2e..d1ddb75cde9b 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -1461,7 +1461,7 @@ public: if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V; Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS); if (isa<FPMathOperator>(BinOp)) - BinOp = setFPAttrs(BinOp, FPMathTag, FMF); + setFPAttrs(BinOp, FPMathTag, FMF); return Insert(BinOp, Name); } @@ -1479,7 +1479,8 @@ public: CallInst *C = CreateIntrinsic(ID, {L->getType()}, {L, R, RoundingV, ExceptV}, nullptr, Name); - return cast<CallInst>(setFPAttrs(C, FPMathTag, UseFMF)); + setFPAttrs(C, FPMathTag, UseFMF); + return C; } Value *CreateNeg(Value *V, const Twine &Name = "", @@ -1504,7 +1505,7 @@ public: MDNode *FPMathTag = nullptr) { if (auto *VC = dyn_cast<Constant>(V)) return Insert(Folder.CreateFNeg(VC), Name); - return Insert(setFPAttrs(BinaryOperator::CreateFNeg(V), FPMathTag, FMF), + return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF), Name); } @@ -1514,9 +1515,7 @@ public: const Twine &Name = "") { if (auto *VC = dyn_cast<Constant>(V)) return Insert(Folder.CreateFNeg(VC), Name); - // TODO: This should return UnaryOperator::CreateFNeg(...) once we are - // confident that they are optimized sufficiently. - return Insert(setFPAttrs(BinaryOperator::CreateFNeg(V), nullptr, + return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr, FMFSource->getFastMathFlags()), Name); } @@ -1534,7 +1533,7 @@ public: return Insert(Folder.CreateUnOp(Opc, VC), Name); Instruction *UnOp = UnaryOperator::Create(Opc, V); if (isa<FPMathOperator>(UnOp)) - UnOp = setFPAttrs(UnOp, FPMathTag, FMF); + setFPAttrs(UnOp, FPMathTag, FMF); return Insert(UnOp, Name); } @@ -1612,19 +1611,19 @@ public: LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, const char *Name) { LoadInst *LI = CreateLoad(Ty, Ptr, Name); - LI->setAlignment(Align); + LI->setAlignment(MaybeAlign(Align)); return LI; } LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, const Twine &Name = "") { LoadInst *LI = CreateLoad(Ty, Ptr, Name); - LI->setAlignment(Align); + LI->setAlignment(MaybeAlign(Align)); return LI; } LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, bool isVolatile, const Twine &Name = "") { LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name); - LI->setAlignment(Align); + LI->setAlignment(MaybeAlign(Align)); return LI; } @@ -1649,7 +1648,7 @@ public: StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align, bool isVolatile = false) { StoreInst *SI = CreateStore(Val, Ptr, isVolatile); - SI->setAlignment(Align); + SI->setAlignment(MaybeAlign(Align)); return SI; } @@ -1913,11 +1912,17 @@ public: return V; } - Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = ""){ + Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui, + V, DestTy, nullptr, Name); return CreateCast(Instruction::FPToUI, V, DestTy, Name); } - Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = ""){ + Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi, + V, DestTy, nullptr, Name); return CreateCast(Instruction::FPToSI, V, DestTy, Name); } @@ -1931,10 +1936,17 @@ public: Value *CreateFPTrunc(Value *V, Type *DestTy, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPCast( + Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr, + Name); return CreateCast(Instruction::FPTrunc, V, DestTy, Name); } Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext, + V, DestTy, nullptr, Name); return CreateCast(Instruction::FPExt, V, DestTy, Name); } @@ -2046,6 +2058,37 @@ public: return Insert(CastInst::CreateFPCast(V, DestTy), Name); } + CallInst *CreateConstrainedFPCast( + Intrinsic::ID ID, Value *V, Type *DestTy, + Instruction *FMFSource = nullptr, const Twine &Name = "", + MDNode *FPMathTag = nullptr, + Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding = None, + Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except = None) { + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C; + switch (ID) { + default: { + Value *RoundingV = getConstrainedFPRounding(Rounding); + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, + nullptr, Name); + } break; + case Intrinsic::experimental_constrained_fpext: + case Intrinsic::experimental_constrained_fptoui: + case Intrinsic::experimental_constrained_fptosi: + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, + Name); + break; + } + if (isa<FPMathOperator>(C)) + setFPAttrs(C, FPMathTag, UseFMF); + return C; + } + // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a // compile time error, instead of converting the string to bool for the // isSigned parameter. @@ -2187,7 +2230,10 @@ public: PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name = "") { - return Insert(PHINode::Create(Ty, NumReservedValues), Name); + PHINode *Phi = PHINode::Create(Ty, NumReservedValues); + if (isa<FPMathOperator>(Phi)) + setFPAttrs(Phi, nullptr /* MDNode* */, FMF); + return Insert(Phi, Name); } CallInst *CreateCall(FunctionType *FTy, Value *Callee, @@ -2195,7 +2241,7 @@ public: MDNode *FPMathTag = nullptr) { CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles); if (isa<FPMathOperator>(CI)) - CI = cast<CallInst>(setFPAttrs(CI, FPMathTag, FMF)); + setFPAttrs(CI, FPMathTag, FMF); return Insert(CI, Name); } @@ -2204,7 +2250,7 @@ public: const Twine &Name = "", MDNode *FPMathTag = nullptr) { CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles); if (isa<FPMathOperator>(CI)) - CI = cast<CallInst>(setFPAttrs(CI, FPMathTag, FMF)); + setFPAttrs(CI, FPMathTag, FMF); return Insert(CI, Name); } @@ -2252,7 +2298,7 @@ public: Sel = addBranchMetadata(Sel, Prof, Unpred); } if (isa<FPMathOperator>(Sel)) - Sel = cast<SelectInst>(setFPAttrs(Sel, nullptr /* MDNode* */, FMF)); + setFPAttrs(Sel, nullptr /* MDNode* */, FMF); return Insert(Sel, Name); } @@ -2454,7 +2500,7 @@ public: } Value *CreatePreserveArrayAccessIndex(Value *Base, unsigned Dimension, - unsigned LastIndex) { + unsigned LastIndex, MDNode *DbgInfo) { assert(isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index."); auto *BaseType = Base->getType(); @@ -2476,6 +2522,8 @@ public: Value *DimV = getInt32(Dimension); CallInst *Fn = CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); return Fn; } @@ -2493,7 +2541,8 @@ public: Value *DIIndex = getInt32(FieldIndex); CallInst *Fn = CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); - Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); return Fn; } @@ -2516,7 +2565,8 @@ public: Value *DIIndex = getInt32(FieldIndex); CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, {Base, GEPIndex, DIIndex}); - Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); return Fn; } diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h index 2aac807623a9..72d8ad1501ae 100644 --- a/include/llvm/IR/InlineAsm.h +++ b/include/llvm/IR/InlineAsm.h @@ -244,6 +244,7 @@ public: Constraint_m, Constraint_o, Constraint_v, + Constraint_A, Constraint_Q, Constraint_R, Constraint_S, diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h index ca419b50da6b..7fb94e9d8c22 100644 --- a/include/llvm/IR/InstrTypes.h +++ b/include/llvm/IR/InstrTypes.h @@ -975,7 +975,7 @@ public: static Type* makeCmpResultType(Type* opnd_type) { if (VectorType* vt = dyn_cast<VectorType>(opnd_type)) { return VectorType::get(Type::getInt1Ty(opnd_type->getContext()), - vt->getNumElements()); + vt->getElementCount()); } return Type::getInt1Ty(opnd_type->getContext()); } @@ -1567,11 +1567,17 @@ public: } /// Extract the alignment of the return value. - unsigned getRetAlignment() const { return Attrs.getRetAlignment(); } + unsigned getRetAlignment() const { + if (const auto MA = Attrs.getRetAlignment()) + return MA->value(); + return 0; + } /// Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned ArgNo) const { - return Attrs.getParamAlignment(ArgNo); + if (const auto MA = Attrs.getParamAlignment(ArgNo)) + return MA->value(); + return 0; } /// Extract the byval type for a call or parameter. diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h index 6a9a74bd16f0..803f6977b32c 100644 --- a/include/llvm/IR/Instruction.h +++ b/include/llvm/IR/Instruction.h @@ -229,6 +229,16 @@ public: return hasMetadataHashEntry(); } + /// Return true if this instruction has the given type of metadata attached. + bool hasMetadata(unsigned KindID) const { + return getMetadata(KindID) != nullptr; + } + + /// Return true if this instruction has the given type of metadata attached. + bool hasMetadata(StringRef Kind) const { + return getMetadata(Kind) != nullptr; + } + /// Get the metadata of given kind attached to this Instruction. /// If the metadata is not found then return null. MDNode *getMetadata(unsigned KindID) const { diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index 215ce45c7b75..fa980df03ef0 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -110,9 +110,11 @@ public: /// Return the alignment of the memory that is being allocated by the /// instruction. unsigned getAlignment() const { - return (1u << (getSubclassDataFromInstruction() & 31)) >> 1; + if (const auto MA = decodeMaybeAlign(getSubclassDataFromInstruction() & 31)) + return MA->value(); + return 0; } - void setAlignment(unsigned Align); + void setAlignment(MaybeAlign Align); /// Return true if this alloca is in the entry block of the function and is a /// constant size. If so, the code generator will fold it into the @@ -182,15 +184,15 @@ public: LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, BasicBlock *InsertAtEnd); LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, - unsigned Align, Instruction *InsertBefore = nullptr); + MaybeAlign Align, Instruction *InsertBefore = nullptr); LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, - unsigned Align, BasicBlock *InsertAtEnd); + MaybeAlign Align, BasicBlock *InsertAtEnd); LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, - unsigned Align, AtomicOrdering Order, + MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr); LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, - unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, + MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); // Deprecated [opaque pointer types] @@ -209,20 +211,20 @@ public: BasicBlock *InsertAtEnd) : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, InsertAtEnd) {} - LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, + LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, MaybeAlign Align, Instruction *InsertBefore = nullptr) : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, InsertBefore) {} - LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, + LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, MaybeAlign Align, BasicBlock *InsertAtEnd) : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, InsertAtEnd) {} - LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, + LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr) : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, Order, SSID, InsertBefore) {} - LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, + LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd) : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, Order, SSID, InsertAtEnd) {} @@ -238,10 +240,13 @@ public: /// Return the alignment of the access that is being performed. unsigned getAlignment() const { - return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1; + if (const auto MA = + decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31)) + return MA->value(); + return 0; } - void setAlignment(unsigned Align); + void setAlignment(MaybeAlign Align); /// Returns the ordering constraint of this load instruction. AtomicOrdering getOrdering() const { @@ -332,17 +337,15 @@ public: StoreInst(Value *Val, Value *Ptr, bool isVolatile = false, Instruction *InsertBefore = nullptr); StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd); - StoreInst(Value *Val, Value *Ptr, bool isVolatile, - unsigned Align, Instruction *InsertBefore = nullptr); - StoreInst(Value *Val, Value *Ptr, bool isVolatile, - unsigned Align, BasicBlock *InsertAtEnd); - StoreInst(Value *Val, Value *Ptr, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SyncScope::ID SSID = SyncScope::System, + StoreInst(Value *Val, Value *Ptr, bool isVolatile, MaybeAlign Align, Instruction *InsertBefore = nullptr); - StoreInst(Value *Val, Value *Ptr, bool isVolatile, - unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, + StoreInst(Value *Val, Value *Ptr, bool isVolatile, MaybeAlign Align, BasicBlock *InsertAtEnd); + StoreInst(Value *Val, Value *Ptr, bool isVolatile, MaybeAlign Align, + AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System, + Instruction *InsertBefore = nullptr); + StoreInst(Value *Val, Value *Ptr, bool isVolatile, MaybeAlign Align, + AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); // allocate space for exactly two operands void *operator new(size_t s) { @@ -363,10 +366,13 @@ public: /// Return the alignment of the access that is being performed unsigned getAlignment() const { - return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1; + if (const auto MA = + decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31)) + return MA->value(); + return 0; } - void setAlignment(unsigned Align); + void setAlignment(MaybeAlign Align); /// Returns the ordering constraint of this store instruction. AtomicOrdering getOrdering() const { @@ -1764,6 +1770,10 @@ public: void setTrueValue(Value *V) { Op<1>() = V; } void setFalseValue(Value *V) { Op<2>() = V; } + /// Swap the true and false values of the select instruction. + /// This doesn't swap prof metadata. + void swapValues() { Op<1>().swap(Op<2>()); } + /// Return a string if the specified operands are invalid /// for a select operation, otherwise return null. static const char *areInvalidOperands(Value *Cond, Value *True, Value *False); @@ -3455,16 +3465,7 @@ public: class SwitchInstProfUpdateWrapper { SwitchInst &SI; Optional<SmallVector<uint32_t, 8> > Weights = None; - - // Sticky invalid state is needed to safely ignore operations with prof data - // in cases where SwitchInstProfUpdateWrapper is created from SwitchInst - // with inconsistent prof data. TODO: once we fix all prof data - // inconsistencies we can turn invalid state to assertions. - enum { - Invalid, - Initialized, - Changed - } State = Invalid; + bool Changed = false; protected: static MDNode *getProfBranchWeightsMD(const SwitchInst &SI); @@ -3482,7 +3483,7 @@ public: SwitchInstProfUpdateWrapper(SwitchInst &SI) : SI(SI) { init(); } ~SwitchInstProfUpdateWrapper() { - if (State == Changed) + if (Changed) SI.setMetadata(LLVMContext::MD_prof, buildProfBranchWeightsMD()); } @@ -3938,6 +3939,9 @@ class CallBrInst : public CallBase { ArrayRef<BasicBlock *> IndirectDests, ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr); + /// Should the Indirect Destinations change, scan + update the Arg list. + void updateArgBlockAddresses(unsigned i, BasicBlock *B); + /// Compute the number of operands to allocate. static int ComputeNumOperands(int NumArgs, int NumIndirectDests, int NumBundleInputs = 0) { @@ -4075,7 +4079,7 @@ public: return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() - 1)); } BasicBlock *getIndirectDest(unsigned i) const { - return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() + i)); + return cast_or_null<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() + i)); } SmallVector<BasicBlock *, 16> getIndirectDests() const { SmallVector<BasicBlock *, 16> IndirectDests; @@ -4087,6 +4091,7 @@ public: *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast<Value *>(B); } void setIndirectDest(unsigned i, BasicBlock *B) { + updateArgBlockAddresses(i, B); *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast<Value *>(B); } @@ -4096,11 +4101,10 @@ public: return i == 0 ? getDefaultDest() : getIndirectDest(i - 1); } - void setSuccessor(unsigned idx, BasicBlock *NewSucc) { - assert(idx < getNumIndirectDests() + 1 && + void setSuccessor(unsigned i, BasicBlock *NewSucc) { + assert(i < getNumIndirectDests() + 1 && "Successor # out of range for callbr!"); - *(&Op<-1>() - getNumIndirectDests() -1 + idx) = - reinterpret_cast<Value *>(NewSucc); + return i == 0 ? setDefaultDest(NewSucc) : setIndirectDest(i - 1, NewSucc); } unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; } @@ -5251,31 +5255,38 @@ public: /// A helper function that returns the pointer operand of a load or store /// instruction. Returns nullptr if not load or store. -inline Value *getLoadStorePointerOperand(Value *V) { +inline const Value *getLoadStorePointerOperand(const Value *V) { if (auto *Load = dyn_cast<LoadInst>(V)) return Load->getPointerOperand(); if (auto *Store = dyn_cast<StoreInst>(V)) return Store->getPointerOperand(); return nullptr; } +inline Value *getLoadStorePointerOperand(Value *V) { + return const_cast<Value *>( + getLoadStorePointerOperand(static_cast<const Value *>(V))); +} /// A helper function that returns the pointer operand of a load, store /// or GEP instruction. Returns nullptr if not load, store, or GEP. -inline Value *getPointerOperand(Value *V) { +inline const Value *getPointerOperand(const Value *V) { if (auto *Ptr = getLoadStorePointerOperand(V)) return Ptr; if (auto *Gep = dyn_cast<GetElementPtrInst>(V)) return Gep->getPointerOperand(); return nullptr; } +inline Value *getPointerOperand(Value *V) { + return const_cast<Value *>(getPointerOperand(static_cast<const Value *>(V))); +} /// A helper function that returns the alignment of load or store instruction. -inline unsigned getLoadStoreAlignment(Value *I) { +inline MaybeAlign getLoadStoreAlignment(Value *I) { assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Expected Load or Store instruction"); if (auto *LI = dyn_cast<LoadInst>(I)) - return LI->getAlignment(); - return cast<StoreInst>(I)->getAlignment(); + return MaybeAlign(LI->getAlignment()); + return MaybeAlign(cast<StoreInst>(I)->getAlignment()); } /// A helper function that returns the address space of the pointer operand of diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h index 438bdb29b706..c989b4a2e72a 100644 --- a/include/llvm/IR/IntrinsicInst.h +++ b/include/llvm/IR/IntrinsicInst.h @@ -259,6 +259,8 @@ namespace llvm { case Intrinsic::experimental_constrained_fdiv: case Intrinsic::experimental_constrained_frem: case Intrinsic::experimental_constrained_fma: + case Intrinsic::experimental_constrained_fptosi: + case Intrinsic::experimental_constrained_fptoui: case Intrinsic::experimental_constrained_fptrunc: case Intrinsic::experimental_constrained_fpext: case Intrinsic::experimental_constrained_sqrt: @@ -271,12 +273,16 @@ namespace llvm { case Intrinsic::experimental_constrained_log: case Intrinsic::experimental_constrained_log10: case Intrinsic::experimental_constrained_log2: + case Intrinsic::experimental_constrained_lrint: + case Intrinsic::experimental_constrained_llrint: case Intrinsic::experimental_constrained_rint: case Intrinsic::experimental_constrained_nearbyint: case Intrinsic::experimental_constrained_maxnum: case Intrinsic::experimental_constrained_minnum: case Intrinsic::experimental_constrained_ceil: case Intrinsic::experimental_constrained_floor: + case Intrinsic::experimental_constrained_lround: + case Intrinsic::experimental_constrained_llround: case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: return true; @@ -405,11 +411,11 @@ namespace llvm { setArgOperand(ARG_DEST, Ptr); } - void setDestAlignment(unsigned Align) { + void setDestAlignment(unsigned Alignment) { removeParamAttr(ARG_DEST, Attribute::Alignment); - if (Align > 0) - addParamAttr(ARG_DEST, - Attribute::getWithAlignment(getContext(), Align)); + if (Alignment > 0) + addParamAttr(ARG_DEST, Attribute::getWithAlignment(getContext(), + Align(Alignment))); } void setLength(Value *L) { @@ -454,11 +460,12 @@ namespace llvm { BaseCL::setArgOperand(ARG_SOURCE, Ptr); } - void setSourceAlignment(unsigned Align) { + void setSourceAlignment(unsigned Alignment) { BaseCL::removeParamAttr(ARG_SOURCE, Attribute::Alignment); - if (Align > 0) - BaseCL::addParamAttr(ARG_SOURCE, Attribute::getWithAlignment( - BaseCL::getContext(), Align)); + if (Alignment > 0) + BaseCL::addParamAttr(ARG_SOURCE, + Attribute::getWithAlignment(BaseCL::getContext(), + Align(Alignment))); } }; diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h index f38f92022d21..9e4ebd915afc 100644 --- a/include/llvm/IR/Intrinsics.h +++ b/include/llvm/IR/Intrinsics.h @@ -100,7 +100,8 @@ namespace Intrinsic { Integer, Vector, Pointer, Struct, Argument, ExtendArgument, TruncArgument, HalfVecArgument, SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt, - VecElementArgument + VecElementArgument, ScalableVecArgument, Subdivide2Argument, + Subdivide4Argument, VecOfBitcastsToInt } Kind; union { @@ -125,14 +126,17 @@ namespace Intrinsic { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || Kind == SameVecWidthArgument || Kind == PtrToArgument || - Kind == PtrToElt || Kind == VecElementArgument); + Kind == PtrToElt || Kind == VecElementArgument || + Kind == Subdivide2Argument || Kind == Subdivide4Argument || + Kind == VecOfBitcastsToInt); return Argument_Info >> 3; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || Kind == SameVecWidthArgument || Kind == PtrToArgument || - Kind == VecElementArgument); + Kind == VecElementArgument || Kind == Subdivide2Argument || + Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt); return (ArgKind)(Argument_Info & 7); } diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index d660f8278437..7a0263f88c2a 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -63,6 +63,12 @@ class NoCapture<int argNo> : IntrinsicProperty { int ArgNo = argNo; } +// NoAlias - The specified argument pointer is not aliasing other "noalias" pointer +// arguments of the intrinsic wrt. the intrinsic scope. +class NoAlias<int argNo> : IntrinsicProperty { + int ArgNo = argNo; +} + // Returned - The specified argument is always the return value of the // intrinsic. class Returned<int argNo> : IntrinsicProperty { @@ -181,6 +187,16 @@ class LLVMVectorElementType<int num> : LLVMMatchType<num>; // vector type, but change the element count to be half as many class LLVMHalfElementsVectorType<int num> : LLVMMatchType<num>; +// Match the type of another intrinsic parameter that is expected to be a +// vector type (i.e. <N x iM>) but with each element subdivided to +// form a vector with more elements that are smaller than the original. +class LLVMSubdivide2VectorType<int num> : LLVMMatchType<num>; +class LLVMSubdivide4VectorType<int num> : LLVMMatchType<num>; + +// Match the element count and bit width of another intrinsic parameter, but +// change the element type to an integer. +class LLVMVectorOfBitcastsToInt<int num> : LLVMMatchType<num>; + def llvm_void_ty : LLVMType<isVoid>; let isAny = 1 in { def llvm_any_ty : LLVMType<Any>; @@ -407,9 +423,9 @@ def int_objc_arc_annotation_bottomup_bbend : Intrinsic<[], //===--------------------- Code Generator Intrinsics ----------------------===// // def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; -def int_addressofreturnaddress : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; -def int_frameaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; -def int_sponentry : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; +def int_addressofreturnaddress : Intrinsic<[llvm_anyptr_ty], [], [IntrNoMem]>; +def int_frameaddress : Intrinsic<[llvm_anyptr_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; +def int_sponentry : Intrinsic<[llvm_anyptr_ty], [], [IntrNoMem]>; def int_read_register : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty], [IntrReadMem], "llvm.read_register">; def int_write_register : Intrinsic<[], [llvm_metadata_ty, llvm_anyint_ty], @@ -451,8 +467,8 @@ def int_thread_pointer : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>, // from being reordered overly much with respect to nearby access to the same // memory while not impeding optimization. def int_prefetch - : Intrinsic<[], [ llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], - [ IntrInaccessibleMemOrArgMemOnly, ReadOnly<0>, NoCapture<0>, + : Intrinsic<[], [ llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], + [ IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, ReadOnly<0>, NoCapture<0>, ImmArg<1>, ImmArg<2>]>; def int_pcmarker : Intrinsic<[], [llvm_i32_ty]>; @@ -460,7 +476,7 @@ def int_readcyclecounter : Intrinsic<[llvm_i64_ty]>; // The assume intrinsic is marked as arbitrarily writing so that proper // control dependencies will be maintained. -def int_assume : Intrinsic<[], [llvm_i1_ty], []>; +def int_assume : Intrinsic<[], [llvm_i1_ty], [IntrWillReturn]>; // Stack Protector Intrinsic - The stackprotector intrinsic writes the stack // guard to the correct place on the stack frame. @@ -493,23 +509,23 @@ def int_instrprof_value_profile : Intrinsic<[], def int_memcpy : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i1_ty], - [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, - WriteOnly<0>, ReadOnly<1>, ImmArg<3>]>; + [IntrArgMemOnly, IntrWillReturn, NoCapture<0>, NoCapture<1>, + NoAlias<0>, NoAlias<1>, WriteOnly<0>, ReadOnly<1>, ImmArg<3>]>; def int_memmove : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i1_ty], - [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, + [IntrArgMemOnly, IntrWillReturn, NoCapture<0>, NoCapture<1>, ReadOnly<1>, ImmArg<3>]>; def int_memset : Intrinsic<[], [llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i1_ty], - [IntrArgMemOnly, NoCapture<0>, WriteOnly<0>, + [IntrArgMemOnly, IntrWillReturn, NoCapture<0>, WriteOnly<0>, ImmArg<3>]>; // FIXME: Add version of these floating point intrinsics which allow non-default // rounding modes and FP exception handling. -let IntrProperties = [IntrNoMem, IntrSpeculatable] in { +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in { def int_fma : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; @@ -551,19 +567,19 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in { def int_minnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative] + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; def int_maxnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative] + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; def int_minimum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative] + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; def int_maximum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative] + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; // NOTE: these are internal interfaces. @@ -576,13 +592,13 @@ def int_siglongjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>; def int_objectsize : Intrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, llvm_i1_ty, llvm_i1_ty, llvm_i1_ty], - [IntrNoMem, IntrSpeculatable, ImmArg<1>, ImmArg<2>, ImmArg<3>]>, + [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<1>, ImmArg<2>, ImmArg<3>]>, GCCBuiltin<"__builtin_object_size">; //===--------------- Constrained Floating Point Intrinsics ----------------===// // -let IntrProperties = [IntrInaccessibleMemOnly] in { +let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { def int_experimental_constrained_fadd : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, @@ -616,6 +632,14 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_fptosi : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty ]>; + + def int_experimental_constrained_fptoui : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty ]>; + def int_experimental_constrained_fptrunc : Intrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty, llvm_metadata_ty, @@ -679,6 +703,14 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { [ LLVMMatchType<0>, llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_lrint : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty, + llvm_metadata_ty ]>; + def int_experimental_constrained_llrint : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty, + llvm_metadata_ty ]>; def int_experimental_constrained_maxnum : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, @@ -697,6 +729,12 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { [ LLVMMatchType<0>, llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_lround : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty ]>; + def int_experimental_constrained_llround : Intrinsic<[ llvm_anyint_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty ]>; def int_experimental_constrained_round : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, llvm_metadata_ty, @@ -706,18 +744,19 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { llvm_metadata_ty, llvm_metadata_ty ]>; } -// FIXME: Add intrinsics for fcmp, fptoui and fptosi. +// FIXME: Add intrinsic for fcmp. +// FIXME: Consider maybe adding intrinsics for sitofp, uitofp. //===------------------------- Expect Intrinsics --------------------------===// // def int_expect : Intrinsic<[llvm_anyint_ty], - [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrWillReturn]>; //===-------------------- Bit Manipulation Intrinsics ---------------------===// // // None of these intrinsics accesses memory at all. -let IntrProperties = [IntrNoMem, IntrSpeculatable] in { +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in { def int_bswap: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_ctpop: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_bitreverse : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; @@ -727,7 +766,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in { [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; } -let IntrProperties = [IntrNoMem, IntrSpeculatable, ImmArg<1>] in { +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<1>] in { def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; def int_cttz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; } @@ -739,7 +778,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, ImmArg<1>] in { // mean the optimizers can change them aggressively. Special handling // needed in a few places. These synthetic intrinsics have no // side-effects and just mark information about their operands. -let IntrProperties = [IntrNoMem, IntrSpeculatable] in { +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in { def int_dbg_declare : Intrinsic<[], [llvm_metadata_ty, llvm_metadata_ty, @@ -796,21 +835,21 @@ def int_eh_sjlj_setup_dispatch : Intrinsic<[], []>; def int_var_annotation : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], - [], "llvm.var.annotation">; + [IntrWillReturn], "llvm.var.annotation">; def int_ptr_annotation : Intrinsic<[LLVMAnyPointerType<llvm_anyint_ty>], [LLVMMatchType<0>, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], - [], "llvm.ptr.annotation">; + [IntrWillReturn], "llvm.ptr.annotation">; def int_annotation : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], - [], "llvm.annotation">; + [IntrWillReturn], "llvm.annotation">; // Annotates the current program point with metadata strings which are emitted // as CodeView debug info records. This is expensive, as it disables inlining // and is modelled as having side effects. def int_codeview_annotation : Intrinsic<[], [llvm_metadata_ty], - [IntrInaccessibleMemOnly, IntrNoDuplicate], + [IntrInaccessibleMemOnly, IntrNoDuplicate, IntrWillReturn], "llvm.codeview.annotation">; //===------------------------ Trampoline Intrinsics -----------------------===// @@ -828,79 +867,77 @@ def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], // // Expose the carry flag from add operations on two integrals. -def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; -def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; - -def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; -def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; - -def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; -def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; - +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in { + def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + + def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + + def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMMatchType<0>, LLVMMatchType<0>]>; +} //===------------------------- Saturation Arithmetic Intrinsics ---------------------===// // def int_sadd_sat : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]>; def int_uadd_sat : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable, Commutative]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]>; def int_ssub_sat : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_usub_sat : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem, IntrSpeculatable]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; //===------------------------- Fixed Point Arithmetic Intrinsics ---------------------===// // def int_smul_fix : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative, ImmArg<2>]>; def int_umul_fix : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative, ImmArg<2>]>; //===------------------- Fixed Point Saturation Arithmetic Intrinsics ----------------===// // def int_smul_fix_sat : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative, ImmArg<2>]>; +def int_umul_fix_sat : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative, ImmArg<2>]>; //===------------------------- Memory Use Markers -------------------------===// // def int_lifetime_start : Intrinsic<[], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; + [IntrArgMemOnly, IntrWillReturn, NoCapture<1>, ImmArg<0>]>; def int_lifetime_end : Intrinsic<[], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; + [IntrArgMemOnly, IntrWillReturn, NoCapture<1>, ImmArg<0>]>; def int_invariant_start : Intrinsic<[llvm_descriptor_ty], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; + [IntrArgMemOnly, IntrWillReturn, NoCapture<1>, ImmArg<0>]>; def int_invariant_end : Intrinsic<[], [llvm_descriptor_ty, llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<2>, ImmArg<1>]>; + [IntrArgMemOnly, IntrWillReturn, NoCapture<2>, ImmArg<1>]>; // launder.invariant.group can't be marked with 'readnone' (IntrNoMem), // because it would cause CSE of two barriers with the same argument. @@ -916,12 +953,12 @@ def int_invariant_end : Intrinsic<[], // might change in the future. def int_launder_invariant_group : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>], - [IntrInaccessibleMemOnly, IntrSpeculatable]>; + [IntrInaccessibleMemOnly, IntrSpeculatable, IntrWillReturn]>; def int_strip_invariant_group : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>], - [IntrSpeculatable, IntrNoMem]>; + [IntrSpeculatable, IntrNoMem, IntrWillReturn]>; //===------------------------ Stackmap Intrinsics -------------------------===// // @@ -964,6 +1001,14 @@ def int_coro_id : Intrinsic<[llvm_token_ty], [llvm_i32_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], [IntrArgMemOnly, IntrReadMem, ReadNone<1>, ReadOnly<2>, NoCapture<2>]>; +def int_coro_id_retcon : Intrinsic<[llvm_token_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, + llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], + []>; +def int_coro_id_retcon_once : Intrinsic<[llvm_token_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, + llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], + []>; def int_coro_alloc : Intrinsic<[llvm_i1_ty], [llvm_token_ty], []>; def int_coro_begin : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_ptr_ty], [WriteOnly<1>]>; @@ -979,6 +1024,13 @@ def int_coro_size : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; def int_coro_save : Intrinsic<[llvm_token_ty], [llvm_ptr_ty], []>; def int_coro_suspend : Intrinsic<[llvm_i8_ty], [llvm_token_ty, llvm_i1_ty], []>; +def int_coro_suspend_retcon : Intrinsic<[llvm_any_ty], [llvm_vararg_ty], []>; +def int_coro_prepare_retcon : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], + [IntrNoMem]>; +def int_coro_alloca_alloc : Intrinsic<[llvm_token_ty], + [llvm_anyint_ty, llvm_i32_ty], []>; +def int_coro_alloca_get : Intrinsic<[llvm_ptr_ty], [llvm_token_ty], []>; +def int_coro_alloca_free : Intrinsic<[], [llvm_token_ty], []>; def int_coro_param : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_ptr_ty], [IntrNoMem, ReadNone<0>, ReadNone<1>]>; @@ -1018,19 +1070,19 @@ def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty], // Supports widenable conditions for guards represented as explicit branches. def int_experimental_widenable_condition : Intrinsic<[llvm_i1_ty], [], - [IntrInaccessibleMemOnly]>; + [IntrInaccessibleMemOnly, IntrWillReturn]>; // NOP: calls/invokes to this intrinsic are removed by codegen -def int_donothing : Intrinsic<[], [], [IntrNoMem]>; +def int_donothing : Intrinsic<[], [], [IntrNoMem, IntrWillReturn]>; // This instruction has no actual effect, though it is treated by the optimizer // has having opaque side effects. This may be inserted into loops to ensure // that they are not removed even if they turn out to be empty, for languages // which specify that infinite loops must be preserved. -def int_sideeffect : Intrinsic<[], [], [IntrInaccessibleMemOnly]>; +def int_sideeffect : Intrinsic<[], [], [IntrInaccessibleMemOnly, IntrWillReturn]>; -// Intrisics to support half precision floating point format -let IntrProperties = [IntrNoMem] in { +// Intrinsics to support half precision floating point format +let IntrProperties = [IntrNoMem, IntrWillReturn] in { def int_convert_to_fp16 : Intrinsic<[llvm_i16_ty], [llvm_anyfloat_ty]>; def int_convert_from_fp16 : Intrinsic<[llvm_anyfloat_ty], [llvm_i16_ty]>; } @@ -1041,7 +1093,11 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], [], "llvm.clear_cache">; // Intrinsic to detect whether its argument is a constant. -def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem], "llvm.is.constant">; +def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem, IntrWillReturn], "llvm.is.constant">; + +// Intrinsic to mask out bits of a pointer. +def int_ptrmask: Intrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty, llvm_anyint_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; //===-------------------------- Masked Intrinsics -------------------------===// // @@ -1049,45 +1105,45 @@ def int_masked_store : Intrinsic<[], [llvm_anyvector_ty, LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [IntrArgMemOnly, ImmArg<2>]>; + [IntrArgMemOnly, IntrWillReturn, ImmArg<2>]>; def int_masked_load : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; + [IntrReadMem, IntrArgMemOnly, IntrWillReturn, ImmArg<1>]>; def int_masked_gather: Intrinsic<[llvm_anyvector_ty], [LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadMem, ImmArg<1>]>; + [IntrReadMem, IntrWillReturn, ImmArg<1>]>; def int_masked_scatter: Intrinsic<[], [llvm_anyvector_ty, LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [ImmArg<2>]>; + [IntrWillReturn, ImmArg<2>]>; def int_masked_expandload: Intrinsic<[llvm_anyvector_ty], [LLVMPointerToElt<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadMem]>; + [IntrReadMem, IntrWillReturn]>; def int_masked_compressstore: Intrinsic<[], [llvm_anyvector_ty, LLVMPointerToElt<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [IntrArgMemOnly]>; + [IntrArgMemOnly, IntrWillReturn]>; // Test whether a pointer is associated with a type metadata identifier. def int_type_test : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_metadata_ty], - [IntrNoMem]>; + [IntrNoMem, IntrWillReturn]>; // Safely loads a function pointer from a virtual table pointer using type metadata. def int_type_checked_load : Intrinsic<[llvm_ptr_ty, llvm_i1_ty], [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty], - [IntrNoMem]>; + [IntrNoMem, IntrWillReturn]>; // Create a branch funnel that implements an indirect call to a limited set of // callees. This needs to be a musttail call. @@ -1098,6 +1154,8 @@ def int_load_relative: Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty], def int_hwasan_check_memaccess : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly, ImmArg<2>]>; +def int_hwasan_check_memaccess_shortgranules : + Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly, ImmArg<2>]>; // Xray intrinsics //===----------------------------------------------------------------------===// @@ -1121,7 +1179,7 @@ def int_memcpy_element_unordered_atomic llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty ], [ - IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>, + IntrArgMemOnly, IntrWillReturn, NoCapture<0>, NoCapture<1>, WriteOnly<0>, ReadOnly<1>, ImmArg<3> ]>; @@ -1132,58 +1190,47 @@ def int_memmove_element_unordered_atomic llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty ], [ - IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>, + IntrArgMemOnly, IntrWillReturn, NoCapture<0>, NoCapture<1>, WriteOnly<0>, ReadOnly<1>, ImmArg<3> ]>; // @llvm.memset.element.unordered.atomic.*(dest, value, length, elementsize) def int_memset_element_unordered_atomic : Intrinsic<[], [ llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i32_ty ], - [ IntrArgMemOnly, NoCapture<0>, WriteOnly<0>, ImmArg<3> ]>; + [ IntrArgMemOnly, IntrWillReturn, NoCapture<0>, WriteOnly<0>, ImmArg<3> ]>; //===------------------------ Reduction Intrinsics ------------------------===// // -def int_experimental_vector_reduce_v2_fadd : Intrinsic<[llvm_anyfloat_ty], - [LLVMMatchType<0>, - llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_v2_fmul : Intrinsic<[llvm_anyfloat_ty], - [LLVMMatchType<0>, - llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_add : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_mul : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_and : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_or : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_xor : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_smax : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_smin : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_umax : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_umin : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_fmax : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_fmin : Intrinsic<[LLVMVectorElementType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; +let IntrProperties = [IntrNoMem, IntrWillReturn] in { + def int_experimental_vector_reduce_v2_fadd : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, + llvm_anyvector_ty]>; + def int_experimental_vector_reduce_v2_fmul : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, + llvm_anyvector_ty]>; + def int_experimental_vector_reduce_add : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_mul : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_and : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_or : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_xor : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_smax : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_smin : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_umax : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_umin : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_fmax : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; + def int_experimental_vector_reduce_fmin : Intrinsic<[LLVMVectorElementType<0>], + [llvm_anyvector_ty]>; +} //===---------- Intrinsics to control hardware supported loops ----------===// diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td index 832aca4fd30f..db01700f409f 100644 --- a/include/llvm/IR/IntrinsicsAArch64.td +++ b/include/llvm/IR/IntrinsicsAArch64.td @@ -691,7 +691,7 @@ def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], // Memory Tagging Extensions (MTE) Intrinsics let TargetPrefix = "aarch64" in { def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], - [IntrInaccessibleMemOnly]>; + [IntrNoMem, IntrHasSideEffects]>; def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrNoMem]>; def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], @@ -707,7 +707,7 @@ def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], // Generate a randomly tagged stack base pointer. def int_aarch64_irg_sp : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty], - [IntrInaccessibleMemOnly]>; + [IntrNoMem, IntrHasSideEffects]>; // Transfer pointer tag with offset. // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where @@ -733,3 +733,124 @@ def int_aarch64_settag_zero : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; } + +// Transactional Memory Extension (TME) Intrinsics +let TargetPrefix = "aarch64" in { +def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">, + Intrinsic<[llvm_i64_ty]>; + +def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>; + +def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">, + Intrinsic<[], [llvm_i64_ty], [ImmArg<0>]>; + +def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">, + Intrinsic<[llvm_i64_ty], [], + [IntrNoMem, IntrHasSideEffects]>; +} + +def llvm_nxv2i1_ty : LLVMType<nxv2i1>; +def llvm_nxv4i1_ty : LLVMType<nxv4i1>; +def llvm_nxv8i1_ty : LLVMType<nxv8i1>; +def llvm_nxv16i1_ty : LLVMType<nxv16i1>; +def llvm_nxv16i8_ty : LLVMType<nxv16i8>; +def llvm_nxv4i32_ty : LLVMType<nxv4i32>; +def llvm_nxv2i64_ty : LLVMType<nxv2i64>; +def llvm_nxv8f16_ty : LLVMType<nxv8f16>; +def llvm_nxv4f32_ty : LLVMType<nxv4f32>; +def llvm_nxv2f64_ty : LLVMType<nxv2f64>; + +let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". + class AdvSIMD_Merged1VectorArg_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_CNT_Intrinsic + : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], + [LLVMVectorOfBitcastsToInt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_Unpack_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMSubdivide2VectorType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_PUNPKHI_Intrinsic + : Intrinsic<[LLVMHalfElementsVectorType<0>], + [llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_DOT_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide4VectorType<0>, + LLVMSubdivide4VectorType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_DOT_Indexed_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide4VectorType<0>, + LLVMSubdivide4VectorType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + // This class of intrinsics are not intended to be useful within LLVM IR but + // are instead here to support some of the more regid parts of the ACLE. + class Builtin_SVCVT<string name, LLVMType OUT, LLVMType IN> + : GCCBuiltin<"__builtin_sve_" # name>, + Intrinsic<[OUT], [OUT, llvm_nxv16i1_ty, IN], [IntrNoMem]>; +} + +//===----------------------------------------------------------------------===// +// SVE + +let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". + +// +// Integer arithmetic +// + +def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; + +def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; +def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; + +def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; +def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; + +// +// Counting bits +// + +def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; + +// +// Permutations and selection +// + +def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; +def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; + +def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; +def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; + +// +// Floating-point comparisons +// + +def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>; + +// +// Predicate operations +// + +def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; +def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; +} diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td index 3982444b5401..ab6ee7f92dd1 100644 --- a/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/include/llvm/IR/IntrinsicsAMDGPU.td @@ -175,6 +175,7 @@ def int_amdgcn_implicit_buffer_ptr : // Set EXEC to the 64-bit value given. // This is always moved to the beginning of the basic block. +// FIXME: Should be mangled for wave size. def int_amdgcn_init_exec : Intrinsic<[], [llvm_i64_ty], // 64-bit literal constant [IntrConvergent, ImmArg<0>]>; @@ -185,7 +186,7 @@ def int_amdgcn_init_exec : Intrinsic<[], def int_amdgcn_init_exec_from_input : Intrinsic<[], [llvm_i32_ty, // 32-bit SGPR input llvm_i32_ty], // bit offset of the thread count - [IntrConvergent]>; + [IntrConvergent, ImmArg<1>]>; def int_amdgcn_wavefrontsize : GCCBuiltin<"__builtin_amdgcn_wavefrontsize">, @@ -199,12 +200,14 @@ def int_amdgcn_wavefrontsize : // The first parameter is s_sendmsg immediate (i16), // the second one is copied to m0 def int_amdgcn_s_sendmsg : GCCBuiltin<"__builtin_amdgcn_s_sendmsg">, - Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, IntrInaccessibleMemOnly]>; + Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], + [ImmArg<0>, IntrNoMem, IntrHasSideEffects]>; def int_amdgcn_s_sendmsghalt : GCCBuiltin<"__builtin_amdgcn_s_sendmsghalt">, - Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, IntrInaccessibleMemOnly]>; + Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], + [ImmArg<0>, IntrNoMem, IntrHasSideEffects]>; def int_amdgcn_s_barrier : GCCBuiltin<"__builtin_amdgcn_s_barrier">, - Intrinsic<[], [], [IntrConvergent]>; + Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent]>; def int_amdgcn_wave_barrier : GCCBuiltin<"__builtin_amdgcn_wave_barrier">, Intrinsic<[], [], [IntrConvergent]>; @@ -835,9 +838,6 @@ defset list<AMDGPUImageDimIntrinsic> AMDGPUImageDimAtomicIntrinsics = { defm int_amdgcn_image_atomic_and : AMDGPUImageDimAtomic<"ATOMIC_AND">; defm int_amdgcn_image_atomic_or : AMDGPUImageDimAtomic<"ATOMIC_OR">; defm int_amdgcn_image_atomic_xor : AMDGPUImageDimAtomic<"ATOMIC_XOR">; - - // TODO: INC/DEC are weird: they seem to have a vdata argument in hardware, - // even though it clearly shouldn't be needed defm int_amdgcn_image_atomic_inc : AMDGPUImageDimAtomic<"ATOMIC_INC">; defm int_amdgcn_image_atomic_dec : AMDGPUImageDimAtomic<"ATOMIC_DEC">; @@ -854,8 +854,8 @@ let TargetPrefix = "amdgcn" in { defset list<AMDGPURsrcIntrinsic> AMDGPUBufferIntrinsics = { -class AMDGPUBufferLoad : Intrinsic < - [llvm_any_ty], +class AMDGPUBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic < + [data_ty], [llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) @@ -863,7 +863,7 @@ class AMDGPUBufferLoad : Intrinsic < llvm_i1_ty], // slc(imm) [IntrReadMem, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; -def int_amdgcn_buffer_load_format : AMDGPUBufferLoad; +def int_amdgcn_buffer_load_format : AMDGPUBufferLoad<llvm_anyfloat_ty>; def int_amdgcn_buffer_load : AMDGPUBufferLoad; def int_amdgcn_s_buffer_load : Intrinsic < @@ -874,9 +874,9 @@ def int_amdgcn_s_buffer_load : Intrinsic < [IntrNoMem, ImmArg<2>]>, AMDGPURsrcIntrinsic<0>; -class AMDGPUBufferStore : Intrinsic < +class AMDGPUBufferStore<LLVMType data_ty = llvm_any_ty> : Intrinsic < [], - [llvm_any_ty, // vdata(VGPR) + [data_ty, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) @@ -884,7 +884,7 @@ class AMDGPUBufferStore : Intrinsic < llvm_i1_ty], // slc(imm) [IntrWriteMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; -def int_amdgcn_buffer_store_format : AMDGPUBufferStore; +def int_amdgcn_buffer_store_format : AMDGPUBufferStore<llvm_anyfloat_ty>; def int_amdgcn_buffer_store : AMDGPUBufferStore; // New buffer intrinsics with separate raw and struct variants. The raw @@ -894,56 +894,68 @@ def int_amdgcn_buffer_store : AMDGPUBufferStore; // and swizzling changes depending on whether idxen is set in the instruction. // These new instrinsics also keep the offset and soffset arguments separate as // they behave differently in bounds checking and swizzling. -class AMDGPURawBufferLoad : Intrinsic < - [llvm_any_ty], +class AMDGPURawBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic < + [data_ty], [llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrReadMem, ImmArg<3>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; -def int_amdgcn_raw_buffer_load_format : AMDGPURawBufferLoad; +def int_amdgcn_raw_buffer_load_format : AMDGPURawBufferLoad<llvm_anyfloat_ty>; def int_amdgcn_raw_buffer_load : AMDGPURawBufferLoad; -class AMDGPUStructBufferLoad : Intrinsic < - [llvm_any_ty], +class AMDGPUStructBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic < + [data_ty], [llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrReadMem, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; -def int_amdgcn_struct_buffer_load_format : AMDGPUStructBufferLoad; +def int_amdgcn_struct_buffer_load_format : AMDGPUStructBufferLoad<llvm_anyfloat_ty>; def int_amdgcn_struct_buffer_load : AMDGPUStructBufferLoad; -class AMDGPURawBufferStore : Intrinsic < +class AMDGPURawBufferStore<LLVMType data_ty = llvm_any_ty> : Intrinsic < [], - [llvm_any_ty, // vdata(VGPR) + [data_ty, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrWriteMem, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; -def int_amdgcn_raw_buffer_store_format : AMDGPURawBufferStore; +def int_amdgcn_raw_buffer_store_format : AMDGPURawBufferStore<llvm_anyfloat_ty>; def int_amdgcn_raw_buffer_store : AMDGPURawBufferStore; -class AMDGPUStructBufferStore : Intrinsic < +class AMDGPUStructBufferStore<LLVMType data_ty = llvm_any_ty> : Intrinsic < [], - [llvm_any_ty, // vdata(VGPR) + [data_ty, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrWriteMem, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; -def int_amdgcn_struct_buffer_store_format : AMDGPUStructBufferStore; +def int_amdgcn_struct_buffer_store_format : AMDGPUStructBufferStore<llvm_anyfloat_ty>; def int_amdgcn_struct_buffer_store : AMDGPUStructBufferStore; -class AMDGPURawBufferAtomic : Intrinsic < - [llvm_anyint_ty], +class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic < + [data_ty], [LLVMMatchType<0>, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) @@ -961,6 +973,8 @@ def int_amdgcn_raw_buffer_atomic_umax : AMDGPURawBufferAtomic; def int_amdgcn_raw_buffer_atomic_and : AMDGPURawBufferAtomic; def int_amdgcn_raw_buffer_atomic_or : AMDGPURawBufferAtomic; def int_amdgcn_raw_buffer_atomic_xor : AMDGPURawBufferAtomic; +def int_amdgcn_raw_buffer_atomic_inc : AMDGPURawBufferAtomic; +def int_amdgcn_raw_buffer_atomic_dec : AMDGPURawBufferAtomic; def int_amdgcn_raw_buffer_atomic_cmpswap : Intrinsic< [llvm_anyint_ty], [LLVMMatchType<0>, // src(VGPR) @@ -972,8 +986,8 @@ def int_amdgcn_raw_buffer_atomic_cmpswap : Intrinsic< [ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<2, 0>; -class AMDGPUStructBufferAtomic : Intrinsic < - [llvm_anyint_ty], +class AMDGPUStructBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic < + [data_ty], [LLVMMatchType<0>, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) @@ -992,6 +1006,8 @@ def int_amdgcn_struct_buffer_atomic_umax : AMDGPUStructBufferAtomic; def int_amdgcn_struct_buffer_atomic_and : AMDGPUStructBufferAtomic; def int_amdgcn_struct_buffer_atomic_or : AMDGPUStructBufferAtomic; def int_amdgcn_struct_buffer_atomic_xor : AMDGPUStructBufferAtomic; +def int_amdgcn_struct_buffer_atomic_inc : AMDGPUStructBufferAtomic; +def int_amdgcn_struct_buffer_atomic_dec : AMDGPUStructBufferAtomic; def int_amdgcn_struct_buffer_atomic_cmpswap : Intrinsic< [llvm_anyint_ty], [LLVMMatchType<0>, // src(VGPR) @@ -1046,7 +1062,10 @@ def int_amdgcn_raw_tbuffer_load : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrReadMem, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; @@ -1057,7 +1076,10 @@ def int_amdgcn_raw_tbuffer_store : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrWriteMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; @@ -1068,7 +1090,10 @@ def int_amdgcn_struct_tbuffer_load : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrReadMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; @@ -1080,7 +1105,10 @@ def int_amdgcn_struct_tbuffer_store : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+), + // swizzled buffer (bit 3 = swz)) [IntrWriteMem, ImmArg<5>, ImmArg<6>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; @@ -1431,6 +1459,13 @@ def int_amdgcn_wqm : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] >; +// Copies the source value to the destination value, such that the source +// is computed as if the entire program were executed in WQM if any other +// program code executes in WQM. +def int_amdgcn_softwqm : Intrinsic<[llvm_any_ty], + [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] +>; + // Return true if at least one thread within the pixel quad passes true into // the function. def int_amdgcn_wqm_vote : Intrinsic<[llvm_i1_ty], @@ -1459,6 +1494,18 @@ def int_amdgcn_set_inactive : LLVMMatchType<0>], // value for the inactive lanes to take [IntrNoMem, IntrConvergent]>; +// Return if the given flat pointer points to a local memory address. +def int_amdgcn_is_shared : GCCBuiltin<"__builtin_amdgcn_is_shared">, + Intrinsic<[llvm_i1_ty], [llvm_ptr_ty], + [IntrNoMem, IntrSpeculatable, NoCapture<0>] +>; + +// Return if the given flat pointer points to a prvate memory address. +def int_amdgcn_is_private : GCCBuiltin<"__builtin_amdgcn_is_private">, + Intrinsic<[llvm_i1_ty], [llvm_ptr_ty], + [IntrNoMem, IntrSpeculatable, NoCapture<0>] +>; + //===----------------------------------------------------------------------===// // CI+ Intrinsics //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td index 4792af097d95..e13da6157e04 100644 --- a/include/llvm/IR/IntrinsicsARM.td +++ b/include/llvm/IR/IntrinsicsARM.td @@ -777,5 +777,14 @@ class Neon_Dot_Intrinsic def int_arm_neon_udot : Neon_Dot_Intrinsic; def int_arm_neon_sdot : Neon_Dot_Intrinsic; +def int_arm_vctp8 : Intrinsic<[llvm_v16i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp16 : Intrinsic<[llvm_v8i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp32 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp64 : Intrinsic<[llvm_v2i1_ty], [llvm_i32_ty], [IntrNoMem]>; + +// GNU eabi mcount +def int_arm_gnu_eabi_mcount : Intrinsic<[], + [], + [IntrReadMem, IntrWriteMem]>; } // end TargetPrefix diff --git a/include/llvm/IR/IntrinsicsBPF.td b/include/llvm/IR/IntrinsicsBPF.td index d7595a2a7700..3618cc6a4128 100644 --- a/include/llvm/IR/IntrinsicsBPF.td +++ b/include/llvm/IR/IntrinsicsBPF.td @@ -20,4 +20,7 @@ let TargetPrefix = "bpf" in { // All intrinsics start with "llvm.bpf." Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; def int_bpf_pseudo : GCCBuiltin<"__builtin_bpf_pseudo">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>; + def int_bpf_preserve_field_info : GCCBuiltin<"__builtin_bpf_preserve_field_info">, + Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<1>]>; } diff --git a/include/llvm/IR/IntrinsicsMips.td b/include/llvm/IR/IntrinsicsMips.td index 6393a9ca35d5..bfcdd80a52d5 100644 --- a/include/llvm/IR/IntrinsicsMips.td +++ b/include/llvm/IR/IntrinsicsMips.td @@ -1260,16 +1260,16 @@ def int_mips_insve_d : GCCBuiltin<"__builtin_msa_insve_d">, def int_mips_ld_b : GCCBuiltin<"__builtin_msa_ld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_h : GCCBuiltin<"__builtin_msa_ld_h">, Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_w : GCCBuiltin<"__builtin_msa_ld_w">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_d : GCCBuiltin<"__builtin_msa_ld_d">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ldi_b : GCCBuiltin<"__builtin_msa_ldi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; @@ -1684,16 +1684,16 @@ def int_mips_srlri_d : GCCBuiltin<"__builtin_msa_srlri_d">, def int_mips_st_b : GCCBuiltin<"__builtin_msa_st_b">, Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly, ImmArg<2>]>; + [IntrArgMemOnly]>; def int_mips_st_h : GCCBuiltin<"__builtin_msa_st_h">, Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly, ImmArg<2>]>; + [IntrArgMemOnly]>; def int_mips_st_w : GCCBuiltin<"__builtin_msa_st_w">, Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly, ImmArg<2>]>; + [IntrArgMemOnly]>; def int_mips_st_d : GCCBuiltin<"__builtin_msa_st_d">, Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly, ImmArg<2>]>; + [IntrArgMemOnly]>; def int_mips_subs_s_b : GCCBuiltin<"__builtin_msa_subs_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; diff --git a/include/llvm/IR/IntrinsicsNVVM.td b/include/llvm/IR/IntrinsicsNVVM.td index dba7dd76c4ff..0483d965ba64 100644 --- a/include/llvm/IR/IntrinsicsNVVM.td +++ b/include/llvm/IR/IntrinsicsNVVM.td @@ -276,6 +276,26 @@ class NVVM_MMA_SUPPORTED<list<WMMA_REGS> frags, string layout_a, string layout_b ); } +class SHFL_INFO<bit sync, string mode, string type, bit return_pred> { + string Suffix = !if(sync, "sync_", "") + # mode # "_" + # type + # !if(return_pred, "p", ""); + + string Name = "int_nvvm_shfl_" # Suffix; + string Builtin = "__nvvm_shfl_" # Suffix; + string IntrName = "llvm.nvvm.shfl." # !subst("_",".", Suffix); + list<int> withGccBuiltin = !if(return_pred, [], [1]); + list<int> withoutGccBuiltin = !if(return_pred, [1], []); + LLVMType OpType = !cond( + !eq(type,"i32"): llvm_i32_ty, + !eq(type,"f32"): llvm_float_ty); + list<LLVMType> RetTy = !if(return_pred, [OpType, llvm_i1_ty], [OpType]); + list<LLVMType> ArgsTy = !if(sync, + [llvm_i32_ty, OpType, llvm_i32_ty, llvm_i32_ty], + [OpType, llvm_i32_ty, llvm_i32_ty]); +} + let TargetPrefix = "nvvm" in { def int_nvvm_prmt : GCCBuiltin<"__nvvm_prmt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], @@ -3955,90 +3975,27 @@ def int_nvvm_read_ptx_sreg_warpsize : PTXReadSRegIntrinsic_r32<"warpsize">; // // SHUFFLE // - -// shfl.down.b32 dest, val, offset, mask_and_clamp -def int_nvvm_shfl_down_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.down.i32">, - GCCBuiltin<"__nvvm_shfl_down_i32">; -def int_nvvm_shfl_down_f32 : - Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.down.f32">, - GCCBuiltin<"__nvvm_shfl_down_f32">; - -// shfl.up.b32 dest, val, offset, mask_and_clamp -def int_nvvm_shfl_up_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.up.i32">, - GCCBuiltin<"__nvvm_shfl_up_i32">; -def int_nvvm_shfl_up_f32 : - Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.up.f32">, - GCCBuiltin<"__nvvm_shfl_up_f32">; - -// shfl.bfly.b32 dest, val, offset, mask_and_clamp -def int_nvvm_shfl_bfly_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.bfly.i32">, - GCCBuiltin<"__nvvm_shfl_bfly_i32">; -def int_nvvm_shfl_bfly_f32 : - Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.bfly.f32">, - GCCBuiltin<"__nvvm_shfl_bfly_f32">; - -// shfl.idx.b32 dest, val, lane, mask_and_clamp -def int_nvvm_shfl_idx_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.idx.i32">, - GCCBuiltin<"__nvvm_shfl_idx_i32">; -def int_nvvm_shfl_idx_f32 : - Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.idx.f32">, - GCCBuiltin<"__nvvm_shfl_idx_f32">; - -// Synchronizing shfl variants available in CUDA-9. -// On sm_70 these don't have to be convergent, so we may eventually want to -// implement non-convergent variant of this intrinsic. - -// shfl.sync.down.b32 dest, threadmask, val, offset , mask_and_clamp -def int_nvvm_shfl_sync_down_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.down.i32">, - GCCBuiltin<"__nvvm_shfl_sync_down_i32">; -def int_nvvm_shfl_sync_down_f32 : - Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.down.f32">, - GCCBuiltin<"__nvvm_shfl_sync_down_f32">; - -// shfl.sync.up.b32 dest, threadmask, val, offset, mask_and_clamp -def int_nvvm_shfl_sync_up_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.up.i32">, - GCCBuiltin<"__nvvm_shfl_sync_up_i32">; -def int_nvvm_shfl_sync_up_f32 : - Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.up.f32">, - GCCBuiltin<"__nvvm_shfl_sync_up_f32">; - -// shfl.sync.bfly.b32 dest, threadmask, val, offset, mask_and_clamp -def int_nvvm_shfl_sync_bfly_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.bfly.i32">, - GCCBuiltin<"__nvvm_shfl_sync_bfly_i32">; -def int_nvvm_shfl_sync_bfly_f32 : - Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.bfly.f32">, - GCCBuiltin<"__nvvm_shfl_sync_bfly_f32">; - -// shfl.sync.idx.b32 dest, threadmask, val, lane, mask_and_clamp -def int_nvvm_shfl_sync_idx_i32 : - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.idx.i32">, - GCCBuiltin<"__nvvm_shfl_sync_idx_i32">; -def int_nvvm_shfl_sync_idx_f32 : - Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.idx.f32">, - GCCBuiltin<"__nvvm_shfl_sync_idx_f32">; +// Generate intrinsics for all variants of shfl instruction. +foreach sync = [0, 1] in { + foreach mode = ["up", "down", "bfly", "idx"] in { + foreach type = ["i32", "f32"] in { + foreach return_pred = [0, 1] in { + foreach i = [SHFL_INFO<sync, mode, type, return_pred>] in { + foreach _ = i.withGccBuiltin in { + def i.Name : GCCBuiltin<i.Builtin>, + Intrinsic<i.RetTy, i.ArgsTy, + [IntrInaccessibleMemOnly, IntrConvergent], + i.IntrName>; + } + foreach _ = i.withoutGccBuiltin in { + def i.Name : Intrinsic<i.RetTy, i.ArgsTy, + [IntrInaccessibleMemOnly, IntrConvergent], i.IntrName>; + } + } + } + } + } +} // // VOTE diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td index 1b892727547d..810979b99934 100644 --- a/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/include/llvm/IR/IntrinsicsWebAssembly.td @@ -24,6 +24,17 @@ def int_wasm_memory_grow : Intrinsic<[llvm_anyint_ty], []>; //===----------------------------------------------------------------------===// +// Trapping float-to-int conversions +//===----------------------------------------------------------------------===// + +def int_wasm_trunc_signed : Intrinsic<[llvm_anyint_ty], + [llvm_anyfloat_ty], + [IntrNoMem]>; +def int_wasm_trunc_unsigned : Intrinsic<[llvm_anyint_ty], + [llvm_anyfloat_ty], + [IntrNoMem]>; + +//===----------------------------------------------------------------------===// // Saturating float-to-int conversions //===----------------------------------------------------------------------===// @@ -89,6 +100,10 @@ def int_wasm_atomic_notify: // SIMD intrinsics //===----------------------------------------------------------------------===// +def int_wasm_swizzle : + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty], + [IntrNoMem, IntrSpeculatable]>; def int_wasm_sub_saturate_signed : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], @@ -109,6 +124,39 @@ def int_wasm_alltrue : Intrinsic<[llvm_i32_ty], [llvm_anyvector_ty], [IntrNoMem, IntrSpeculatable]>; +def int_wasm_qfma : + Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_qfms : + Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_narrow_signed : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_narrow_unsigned : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_widen_low_signed : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_widen_high_signed : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_widen_low_unsigned : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty], + [IntrNoMem, IntrSpeculatable]>; +def int_wasm_widen_high_unsigned : + Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty], + [IntrNoMem, IntrSpeculatable]>; + //===----------------------------------------------------------------------===// // Bulk memory intrinsics @@ -133,4 +181,14 @@ def int_wasm_tls_size : [], [IntrNoMem, IntrSpeculatable]>; +def int_wasm_tls_align : + Intrinsic<[llvm_anyint_ty], + [], + [IntrNoMem, IntrSpeculatable]>; + +def int_wasm_tls_base : + Intrinsic<[llvm_ptr_ty], + [], + [IntrReadMem]>; + } // TargetPrefix = "wasm" diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td index 236d312d7d78..5796686dd79f 100644 --- a/include/llvm/IR/IntrinsicsX86.td +++ b/include/llvm/IR/IntrinsicsX86.td @@ -2091,16 +2091,20 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_ptr_ty], [], []>; def int_x86_lwpins32 : GCCBuiltin<"__builtin_ia32_lwpins32">, - Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [ImmArg<2>]>; def int_x86_lwpins64 : GCCBuiltin<"__builtin_ia32_lwpins64">, - Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], + [ImmArg<2>]>; def int_x86_lwpval32 : GCCBuiltin<"__builtin_ia32_lwpval32">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [ImmArg<2>]>; def int_x86_lwpval64 : GCCBuiltin<"__builtin_ia32_lwpval64">, - Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], + [ImmArg<2>]>; } //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h index c80504500418..91bd57dc5ac0 100644 --- a/include/llvm/IR/LLVMContext.h +++ b/include/llvm/IR/LLVMContext.h @@ -72,34 +72,9 @@ public: // Pinned metadata names, which always have the same value. This is a // compile-time performance optimization, not a correctness optimization. enum : unsigned { - MD_dbg = 0, // "dbg" - MD_tbaa = 1, // "tbaa" - MD_prof = 2, // "prof" - MD_fpmath = 3, // "fpmath" - MD_range = 4, // "range" - MD_tbaa_struct = 5, // "tbaa.struct" - MD_invariant_load = 6, // "invariant.load" - MD_alias_scope = 7, // "alias.scope" - MD_noalias = 8, // "noalias", - MD_nontemporal = 9, // "nontemporal" - MD_mem_parallel_loop_access = 10, // "llvm.mem.parallel_loop_access" - MD_nonnull = 11, // "nonnull" - MD_dereferenceable = 12, // "dereferenceable" - MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" - MD_make_implicit = 14, // "make.implicit" - MD_unpredictable = 15, // "unpredictable" - MD_invariant_group = 16, // "invariant.group" - MD_align = 17, // "align" - MD_loop = 18, // "llvm.loop" - MD_type = 19, // "type" - MD_section_prefix = 20, // "section_prefix" - MD_absolute_symbol = 21, // "absolute_symbol" - MD_associated = 22, // "associated" - MD_callees = 23, // "callees" - MD_irr_loop = 24, // "irr_loop" - MD_access_group = 25, // "llvm.access.group" - MD_callback = 26, // "callback" - MD_preserve_access_index = 27, // "llvm.preserve.*.access.index" +#define LLVM_FIXED_MD_KIND(EnumID, Name, Value) EnumID = Value, +#include "llvm/IR/FixedMetadataKinds.def" +#undef LLVM_FIXED_MD_KIND }; /// Known operand bundle tag IDs, which always have the same value. All diff --git a/include/llvm/IR/MDBuilder.h b/include/llvm/IR/MDBuilder.h index 3a2b1bddf45d..11e2e2623257 100644 --- a/include/llvm/IR/MDBuilder.h +++ b/include/llvm/IR/MDBuilder.h @@ -16,6 +16,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/GlobalValue.h" #include "llvm/Support/DataTypes.h" #include <utility> @@ -75,6 +76,10 @@ public: /// Return metadata containing the section prefix for a function. MDNode *createFunctionSectionPrefix(StringRef Prefix); + /// return metadata containing expected value + MDNode *createMisExpect(uint64_t Index, uint64_t LikelyWeight, + uint64_t UnlikelyWeight); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h index 7ca2540181ba..f62b1e246cca 100644 --- a/include/llvm/IR/Metadata.h +++ b/include/llvm/IR/Metadata.h @@ -601,7 +601,7 @@ dyn_extract_or_null(Y &&MD) { /// These are used to efficiently contain a byte sequence for metadata. /// MDString is always unnamed. class MDString : public Metadata { - friend class StringMapEntry<MDString>; + friend class StringMapEntryStorage<MDString>; StringMapEntry<MDString> *Entry = nullptr; @@ -806,7 +806,7 @@ public: /// Ensure that this has RAUW support, and then return it. ReplaceableMetadataImpl *getOrCreateReplaceableUses() { if (!hasReplaceableUses()) - makeReplaceable(llvm::make_unique<ReplaceableMetadataImpl>(getContext())); + makeReplaceable(std::make_unique<ReplaceableMetadataImpl>(getContext())); return getReplaceableUses(); } diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h index f458680cfe15..59331142766a 100644 --- a/include/llvm/IR/Module.h +++ b/include/llvm/IR/Module.h @@ -46,6 +46,7 @@ class FunctionType; class GVMaterializer; class LLVMContext; class MemoryBuffer; +class Pass; class RandomNumberGenerator; template <class PtrType> class SmallPtrSetImpl; class StructType; diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h index aacf8cfc089f..be60447abd87 100644 --- a/include/llvm/IR/ModuleSummaryIndex.h +++ b/include/llvm/IR/ModuleSummaryIndex.h @@ -119,7 +119,7 @@ class GlobalValueSummary; using GlobalValueSummaryList = std::vector<std::unique_ptr<GlobalValueSummary>>; -struct LLVM_ALIGNAS(8) GlobalValueSummaryInfo { +struct alignas(8) GlobalValueSummaryInfo { union NameOrGV { NameOrGV(bool HaveGVs) { if (HaveGVs) @@ -603,7 +603,7 @@ public: if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() || !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() || !TypeCheckedLoadConstVCalls.empty()) - TIdInfo = llvm::make_unique<TypeIdInfo>(TypeIdInfo{ + TIdInfo = std::make_unique<TypeIdInfo>(TypeIdInfo{ std::move(TypeTests), std::move(TypeTestAssumeVCalls), std::move(TypeCheckedLoadVCalls), std::move(TypeTestAssumeConstVCalls), @@ -632,6 +632,8 @@ public: /// Return the list of <CalleeValueInfo, CalleeInfo> pairs. ArrayRef<EdgeTy> calls() const { return CallGraphEdgeList; } + void addCall(EdgeTy E) { CallGraphEdgeList.push_back(E); } + /// Returns the list of type identifiers used by this function in /// llvm.type.test intrinsics other than by an llvm.assume intrinsic, /// represented as GUIDs. @@ -680,7 +682,7 @@ public: /// were unable to devirtualize a checked call. void addTypeTest(GlobalValue::GUID Guid) { if (!TIdInfo) - TIdInfo = llvm::make_unique<TypeIdInfo>(); + TIdInfo = std::make_unique<TypeIdInfo>(); TIdInfo->TypeTests.push_back(Guid); } @@ -780,7 +782,7 @@ public: void setVTableFuncs(VTableFuncList Funcs) { assert(!VTableFuncs); - VTableFuncs = llvm::make_unique<VTableFuncList>(std::move(Funcs)); + VTableFuncs = std::make_unique<VTableFuncList>(std::move(Funcs)); } ArrayRef<VirtFuncOffset> vTableFuncs() const { @@ -1293,6 +1295,12 @@ public: return nullptr; } + TypeIdSummary *getTypeIdSummary(StringRef TypeId) { + return const_cast<TypeIdSummary *>( + static_cast<const ModuleSummaryIndex *>(this)->getTypeIdSummary( + TypeId)); + } + const std::map<std::string, TypeIdCompatibleVtableInfo> & typeIdCompatibleVtableMap() const { return TypeIdCompatibleVtableMap; @@ -1411,7 +1419,7 @@ template <> struct GraphTraits<ModuleSummaryIndex *> : public GraphTraits<ValueInfo> { static NodeRef getEntryNode(ModuleSummaryIndex *I) { std::unique_ptr<GlobalValueSummary> Root = - make_unique<FunctionSummary>(I->calculateCallGraphRoot()); + std::make_unique<FunctionSummary>(I->calculateCallGraphRoot()); GlobalValueSummaryInfo G(I->haveGVs()); G.SummaryList.push_back(std::move(Root)); static auto P = diff --git a/include/llvm/IR/ModuleSummaryIndexYAML.h b/include/llvm/IR/ModuleSummaryIndexYAML.h index 26d9c43fabf1..4d4a67c75172 100644 --- a/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -220,7 +220,7 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { V.emplace(RefGUID, /*IsAnalysis=*/false); Refs.push_back(ValueInfo(/*IsAnalysis=*/false, &*V.find(RefGUID))); } - Elem.SummaryList.push_back(llvm::make_unique<FunctionSummary>( + Elem.SummaryList.push_back(std::make_unique<FunctionSummary>( GlobalValueSummary::GVFlags( static_cast<GlobalValue::LinkageTypes>(FSum.Linkage), FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal, FSum.CanAutoHide), diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h index 8199c65ca8a0..037f5aed03ee 100644 --- a/include/llvm/IR/Operator.h +++ b/include/llvm/IR/Operator.h @@ -379,16 +379,25 @@ public: return false; switch (Opcode) { + case Instruction::FNeg: + case Instruction::FAdd: + case Instruction::FSub: + case Instruction::FMul: + case Instruction::FDiv: + case Instruction::FRem: + // FIXME: To clean up and correct the semantics of fast-math-flags, FCmp + // should not be treated as a math op, but the other opcodes should. + // This would make things consistent with Select/PHI (FP value type + // determines whether they are math ops and, therefore, capable of + // having fast-math-flags). case Instruction::FCmp: return true; - // non math FP Operators (no FMF) - case Instruction::ExtractElement: - case Instruction::ShuffleVector: - case Instruction::InsertElement: case Instruction::PHI: - return false; - default: + case Instruction::Select: + case Instruction::Call: return V->getType()->isFPOrFPVectorTy(); + default: + return false; } } }; diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h index 37fe2a5b01ad..1e1f4a92f844 100644 --- a/include/llvm/IR/PassManager.h +++ b/include/llvm/IR/PassManager.h @@ -45,6 +45,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManagerInternal.h" +#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/TypeName.h" #include "llvm/Support/raw_ostream.h" @@ -418,7 +419,7 @@ template <typename PassT, typename IRUnitT, typename AnalysisManagerT, typename PassT::Result getAnalysisResultUnpackTuple(AnalysisManagerT &AM, IRUnitT &IR, std::tuple<ArgTs...> Args, - llvm::index_sequence<Ns...>) { + std::index_sequence<Ns...>) { (void)Args; return AM.template getResult<PassT>(IR, std::get<Ns>(Args)...); } @@ -435,7 +436,7 @@ getAnalysisResult(AnalysisManager<IRUnitT, AnalysisArgTs...> &AM, IRUnitT &IR, std::tuple<MainArgTs...> Args) { return (getAnalysisResultUnpackTuple< PassT, IRUnitT>)(AM, IR, Args, - llvm::index_sequence_for<AnalysisArgTs...>{}); + std::index_sequence_for<AnalysisArgTs...>{}); } } // namespace detail diff --git a/include/llvm/IR/PassManagerInternal.h b/include/llvm/IR/PassManagerInternal.h index 58198bf67b11..c602c0b5cc20 100644 --- a/include/llvm/IR/PassManagerInternal.h +++ b/include/llvm/IR/PassManagerInternal.h @@ -289,7 +289,7 @@ struct AnalysisPassModel : AnalysisPassConcept<IRUnitT, PreservedAnalysesT, AnalysisResultConcept<IRUnitT, PreservedAnalysesT, InvalidatorT>> run(IRUnitT &IR, AnalysisManager<IRUnitT, ExtraArgTs...> &AM, ExtraArgTs... ExtraArgs) override { - return llvm::make_unique<ResultModelT>( + return std::make_unique<ResultModelT>( Pass.run(IR, AM, std::forward<ExtraArgTs>(ExtraArgs)...)); } diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h index 0f03d7cc56b8..2851b24c05ae 100644 --- a/include/llvm/IR/PatternMatch.h +++ b/include/llvm/IR/PatternMatch.h @@ -88,6 +88,25 @@ inline class_match<UndefValue> m_Undef() { return class_match<UndefValue>(); } /// Match an arbitrary Constant and ignore it. inline class_match<Constant> m_Constant() { return class_match<Constant>(); } +/// Match an arbitrary basic block value and ignore it. +inline class_match<BasicBlock> m_BasicBlock() { + return class_match<BasicBlock>(); +} + +/// Inverting matcher +template <typename Ty> struct match_unless { + Ty M; + + match_unless(const Ty &Matcher) : M(Matcher) {} + + template <typename ITy> bool match(ITy *V) { return !M.match(V); } +}; + +/// Match if the inner matcher does *NOT* match. +template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { + return match_unless<Ty>(M); +} + /// Matching combinators template <typename LTy, typename RTy> struct match_combine_or { LTy L; @@ -300,6 +319,15 @@ template <typename Predicate> struct cstfp_pred_ty : public Predicate { // /////////////////////////////////////////////////////////////////////////////// +struct is_any_apint { + bool isValue(const APInt &C) { return true; } +}; +/// Match an integer or vector with any integral constant. +/// For vectors, this includes constants with undefined elements. +inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { + return cst_pred_ty<is_any_apint>(); +} + struct is_all_ones { bool isValue(const APInt &C) { return C.isAllOnesValue(); } }; @@ -388,6 +416,18 @@ inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; } +struct is_negated_power2 { + bool isValue(const APInt &C) { return (-C).isPowerOf2(); } +}; +/// Match a integer or vector negated power-of-2. +/// For vectors, this includes constants with undefined elements. +inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { + return cst_pred_ty<is_negated_power2>(); +} +inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { + return V; +} + struct is_power2_or_zero { bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } }; @@ -528,6 +568,12 @@ inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } /// Match a ConstantFP, capturing the value if we match. inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } +/// Match a basic block value, capturing it if we match. +inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } +inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { + return V; +} + /// Match a specified Value*. struct specificval_ty { const Value *Val; @@ -597,11 +643,11 @@ struct bind_const_intval_ty { }; /// Match a specified integer value or vector of all elements of that -// value. +/// value. struct specific_intval { - uint64_t Val; + APInt Val; - specific_intval(uint64_t V) : Val(V) {} + specific_intval(APInt V) : Val(std::move(V)) {} template <typename ITy> bool match(ITy *V) { const auto *CI = dyn_cast<ConstantInt>(V); @@ -609,18 +655,50 @@ struct specific_intval { if (const auto *C = dyn_cast<Constant>(V)) CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue()); - return CI && CI->getValue() == Val; + return CI && APInt::isSameValue(CI->getValue(), Val); } }; /// Match a specific integer value or vector with all elements equal to /// the value. -inline specific_intval m_SpecificInt(uint64_t V) { return specific_intval(V); } +inline specific_intval m_SpecificInt(APInt V) { + return specific_intval(std::move(V)); +} + +inline specific_intval m_SpecificInt(uint64_t V) { + return m_SpecificInt(APInt(64, V)); +} /// Match a ConstantInt and bind to its value. This does not match /// ConstantInts wider than 64-bits. inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } +/// Match a specified basic block value. +struct specific_bbval { + BasicBlock *Val; + + specific_bbval(BasicBlock *Val) : Val(Val) {} + + template <typename ITy> bool match(ITy *V) { + const auto *BB = dyn_cast<BasicBlock>(V); + return BB && BB == Val; + } +}; + +/// Match a specific basic block value. +inline specific_bbval m_SpecificBB(BasicBlock *BB) { + return specific_bbval(BB); +} + +/// A commutative-friendly version of m_Specific(). +inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { + return BB; +} +inline deferredval_ty<const BasicBlock> +m_Deferred(const BasicBlock *const &BB) { + return BB; +} + //===----------------------------------------------------------------------===// // Matcher for any binary operator. // @@ -968,6 +1046,12 @@ struct is_idiv_op { } }; +struct is_irem_op { + bool isOpType(unsigned Opcode) { + return Opcode == Instruction::SRem || Opcode == Instruction::URem; + } +}; + /// Matches shift operations. template <typename LHS, typename RHS> inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, @@ -1003,6 +1087,13 @@ inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); } +/// Matches integer remainder operations. +template <typename LHS, typename RHS> +inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, + const RHS &R) { + return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); +} + //===----------------------------------------------------------------------===// // Class that matches exact binary ops. // @@ -1210,6 +1301,12 @@ inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { return CastClass_match<OpTy, Instruction::Trunc>(Op); } +template <typename OpTy> +inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> +m_TruncOrSelf(const OpTy &Op) { + return m_CombineOr(m_Trunc(Op), Op); +} + /// Matches SExt. template <typename OpTy> inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { @@ -1223,12 +1320,33 @@ inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { } template <typename OpTy> +inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> +m_ZExtOrSelf(const OpTy &Op) { + return m_CombineOr(m_ZExt(Op), Op); +} + +template <typename OpTy> +inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> +m_SExtOrSelf(const OpTy &Op) { + return m_CombineOr(m_SExt(Op), Op); +} + +template <typename OpTy> inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, CastClass_match<OpTy, Instruction::SExt>> m_ZExtOrSExt(const OpTy &Op) { return m_CombineOr(m_ZExt(Op), m_SExt(Op)); } +template <typename OpTy> +inline match_combine_or< + match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, + CastClass_match<OpTy, Instruction::SExt>>, + OpTy> +m_ZExtOrSExtOrSelf(const OpTy &Op) { + return m_CombineOr(m_ZExtOrSExt(Op), Op); +} + /// Matches UIToFP. template <typename OpTy> inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { @@ -1274,27 +1392,34 @@ struct br_match { inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } -template <typename Cond_t> struct brc_match { +template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> +struct brc_match { Cond_t Cond; - BasicBlock *&T, *&F; + TrueBlock_t T; + FalseBlock_t F; - brc_match(const Cond_t &C, BasicBlock *&t, BasicBlock *&f) + brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) : Cond(C), T(t), F(f) {} template <typename OpTy> bool match(OpTy *V) { if (auto *BI = dyn_cast<BranchInst>(V)) - if (BI->isConditional() && Cond.match(BI->getCondition())) { - T = BI->getSuccessor(0); - F = BI->getSuccessor(1); - return true; - } + if (BI->isConditional() && Cond.match(BI->getCondition())) + return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); return false; } }; template <typename Cond_t> -inline brc_match<Cond_t> m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { - return brc_match<Cond_t>(C, T, F); +inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> +m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { + return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( + C, m_BasicBlock(T), m_BasicBlock(F)); +} + +template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> +inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> +m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { + return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); } //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/RemarkStreamer.h b/include/llvm/IR/RemarkStreamer.h index f34cc660b2fb..2abf6f99cb08 100644 --- a/include/llvm/IR/RemarkStreamer.h +++ b/include/llvm/IR/RemarkStreamer.h @@ -25,12 +25,12 @@ namespace llvm { /// Streamer for remarks. class RemarkStreamer { - /// The filename that the remark diagnostics are emitted to. - const std::string Filename; /// The regex used to filter remarks based on the passes that emit them. Optional<Regex> PassFilter; /// The object used to serialize the remarks to a specific format. - std::unique_ptr<remarks::Serializer> Serializer; + std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer; + /// The filename that the remark diagnostics are emitted to. + const Optional<std::string> Filename; /// Convert diagnostics into remark objects. /// The lifetime of the members of the result is bound to the lifetime of @@ -38,14 +38,16 @@ class RemarkStreamer { remarks::Remark toRemark(const DiagnosticInfoOptimizationBase &Diag); public: - RemarkStreamer(StringRef Filename, - std::unique_ptr<remarks::Serializer> Serializer); + RemarkStreamer(std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer, + Optional<StringRef> Filename = None); /// Return the filename that the remark diagnostics are emitted to. - StringRef getFilename() const { return Filename; } + Optional<StringRef> getFilename() const { + return Filename ? Optional<StringRef>(*Filename) : None; + } /// Return stream that the remark diagnostics are emitted to. - raw_ostream &getStream() { return Serializer->OS; } + raw_ostream &getStream() { return RemarkSerializer->OS; } /// Return the serializer used for this stream. - remarks::Serializer &getSerializer() { return *Serializer; } + remarks::RemarkSerializer &getSerializer() { return *RemarkSerializer; } /// Set a pass filter based on a regex \p Filter. /// Returns an error if the regex is invalid. Error setFilter(StringRef Filter); @@ -84,13 +86,21 @@ struct RemarkSetupFormatError : RemarkSetupErrorInfo<RemarkSetupFormatError> { using RemarkSetupErrorInfo<RemarkSetupFormatError>::RemarkSetupErrorInfo; }; -/// Setup optimization remarks. +/// Setup optimization remarks that output to a file. Expected<std::unique_ptr<ToolOutputFile>> setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, unsigned RemarksHotnessThreshold = 0); +/// Setup optimization remarks that output directly to a raw_ostream. +/// \p OS is managed by the caller and should be open for writing as long as \p +/// Context is streaming remarks to it. +Error setupOptimizationRemarks(LLVMContext &Context, raw_ostream &OS, + StringRef RemarksPasses, StringRef RemarksFormat, + bool RemarksWithHotness, + unsigned RemarksHotnessThreshold = 0); + } // end namespace llvm #endif // LLVM_IR_REMARKSTREAMER_H diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h index f2aa49030aaa..d0961dac833d 100644 --- a/include/llvm/IR/Type.h +++ b/include/llvm/IR/Type.h @@ -21,6 +21,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TypeSize.h" #include <cassert> #include <cstdint> #include <iterator> @@ -281,12 +282,15 @@ public: /// This will return zero if the type does not have a size or is not a /// primitive type. /// + /// If this is a scalable vector type, the scalable property will be set and + /// the runtime size will be a positive integer multiple of the base size. + /// /// Note that this may not reflect the size of memory allocated for an /// instance of the type or the number of bytes that are written when an /// instance of the type is stored to memory. The DataLayout class provides /// additional query functions to provide this information. /// - unsigned getPrimitiveSizeInBits() const LLVM_READONLY; + TypeSize getPrimitiveSizeInBits() const LLVM_READONLY; /// If this is a vector type, return the getPrimitiveSizeInBits value for the /// element type. Otherwise return the getPrimitiveSizeInBits value for this @@ -368,6 +372,7 @@ public: inline bool getVectorIsScalable() const; inline unsigned getVectorNumElements() const; + inline ElementCount getVectorElementCount() const; Type *getVectorElementType() const { assert(getTypeID() == VectorTyID); return ContainedTys[0]; @@ -378,6 +383,14 @@ public: return ContainedTys[0]; } + /// Given an integer or vector type, change the lane bitwidth to NewBitwidth, + /// whilst keeping the old number of lanes. + inline Type *getWithNewBitWidth(unsigned NewBitWidth) const; + + /// Given scalar/vector integer type, returns a type with elements twice as + /// wide as in the original type. For vectors, preserves element count. + inline Type *getExtendedType() const; + /// Get the address space of this pointer or pointer vector type. inline unsigned getPointerAddressSpace() const; diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h index 19d87c5c621d..850ee72a0387 100644 --- a/include/llvm/IR/User.h +++ b/include/llvm/IR/User.h @@ -111,7 +111,7 @@ public: #endif } /// Placement delete - required by std, called if the ctor throws. - void operator delete(void *Usr, unsigned, bool) { + void operator delete(void *Usr, unsigned, unsigned) { // Note: If a subclass manipulates the information which is required to calculate the // Usr memory pointer, e.g. NumUserOperands, the operator delete of that subclass has // to restore the changed information to the original value, since the dtor of that class diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h index b2d8e7ac4741..f2c4b3b3f203 100644 --- a/include/llvm/IR/Value.h +++ b/include/llvm/IR/Value.h @@ -14,8 +14,10 @@ #define LLVM_IR_VALUE_H #include "llvm-c/Types.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Use.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Casting.h" #include <cassert> @@ -292,10 +294,29 @@ public: /// "V" instead of "this". This function skips metadata entries in the list. void replaceNonMetadataUsesWith(Value *V); + /// Go through the uses list for this definition and make each use point + /// to "V" if the callback ShouldReplace returns true for the given Use. + /// Unlike replaceAllUsesWith() this function does not support basic block + /// values or constant users. + void replaceUsesWithIf(Value *New, + llvm::function_ref<bool(Use &U)> ShouldReplace) { + assert(New && "Value::replaceUsesWithIf(<null>) is invalid!"); + assert(New->getType() == getType() && + "replaceUses of value with new value of different type!"); + + for (use_iterator UI = use_begin(), E = use_end(); UI != E;) { + Use &U = *UI; + ++UI; + if (!ShouldReplace(U)) + continue; + U.set(New); + } + } + /// replaceUsesOutsideBlock - Go through the uses list for this definition and /// make each use point to "V" instead of "this" when the use is outside the /// block. 'This's use list is expected to have at least one element. - /// Unlike replaceAllUsesWith this function does not support basic block + /// Unlike replaceAllUsesWith() this function does not support basic block /// values or constant users. void replaceUsesOutsideBlock(Value *V, BasicBlock *BB); @@ -493,17 +514,27 @@ public: /// swifterror attribute. bool isSwiftError() const; - /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. + /// Strip off pointer casts, all-zero GEPs and address space casts. /// /// Returns the original uncasted value. If this is called on a non-pointer /// value, it returns 'this'. const Value *stripPointerCasts() const; Value *stripPointerCasts() { return const_cast<Value *>( - static_cast<const Value *>(this)->stripPointerCasts()); + static_cast<const Value *>(this)->stripPointerCasts()); } - /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases + /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. + /// + /// Returns the original uncasted value. If this is called on a non-pointer + /// value, it returns 'this'. + const Value *stripPointerCastsAndAliases() const; + Value *stripPointerCastsAndAliases() { + return const_cast<Value *>( + static_cast<const Value *>(this)->stripPointerCastsAndAliases()); + } + + /// Strip off pointer casts, all-zero GEPs and address space casts /// but ensures the representation of the result stays the same. /// /// Returns the original uncasted value with the same representation. If this @@ -514,26 +545,15 @@ public: ->stripPointerCastsSameRepresentation()); } - /// Strip off pointer casts, all-zero GEPs, aliases and invariant group - /// info. + /// Strip off pointer casts, all-zero GEPs and invariant group info. /// /// Returns the original uncasted value. If this is called on a non-pointer /// value, it returns 'this'. This function should be used only in /// Alias analysis. const Value *stripPointerCastsAndInvariantGroups() const; Value *stripPointerCastsAndInvariantGroups() { - return const_cast<Value *>( - static_cast<const Value *>(this)->stripPointerCastsAndInvariantGroups()); - } - - /// Strip off pointer casts and all-zero GEPs. - /// - /// Returns the original uncasted value. If this is called on a non-pointer - /// value, it returns 'this'. - const Value *stripPointerCastsNoFollowAliases() const; - Value *stripPointerCastsNoFollowAliases() { - return const_cast<Value *>( - static_cast<const Value *>(this)->stripPointerCastsNoFollowAliases()); + return const_cast<Value *>(static_cast<const Value *>(this) + ->stripPointerCastsAndInvariantGroups()); } /// Strip off pointer casts and all-constant inbounds GEPs. @@ -612,7 +632,7 @@ public: /// /// Returns an alignment which is either specified explicitly, e.g. via /// align attribute of a function argument, or guaranteed by DataLayout. - unsigned getPointerAlignment(const DataLayout &DL) const; + MaybeAlign getPointerAlignment(const DataLayout &DL) const; /// Translate PHI node to its predecessor from the given basic block. /// diff --git a/include/llvm/IR/ValueMap.h b/include/llvm/IR/ValueMap.h index 6a79b1d387f3..fb5440d5efe8 100644 --- a/include/llvm/IR/ValueMap.h +++ b/include/llvm/IR/ValueMap.h @@ -33,11 +33,11 @@ #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/UniqueLock.h" #include <algorithm> #include <cassert> #include <cstddef> #include <iterator> +#include <mutex> #include <type_traits> #include <utility> @@ -93,7 +93,6 @@ class ValueMap { MapT Map; Optional<MDMapT> MDMap; ExtraData Data; - bool MayMapMetadata = true; public: using key_type = KeyT; @@ -120,10 +119,6 @@ public: } Optional<MDMapT> &getMDMap() { return MDMap; } - bool mayMapMetadata() const { return MayMapMetadata; } - void enableMapMetadata() { MayMapMetadata = true; } - void disableMapMetadata() { MayMapMetadata = false; } - /// Get the mapped metadata, if it's in the map. Optional<Metadata *> getMappedMD(const Metadata *MD) const { if (!MDMap) @@ -266,9 +261,9 @@ public: // Make a copy that won't get changed even when *this is destroyed. ValueMapCallbackVH Copy(*this); typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data); - unique_lock<typename Config::mutex_type> Guard; + std::unique_lock<typename Config::mutex_type> Guard; if (M) - Guard = unique_lock<typename Config::mutex_type>(*M); + Guard = std::unique_lock<typename Config::mutex_type>(*M); Config::onDelete(Copy.Map->Data, Copy.Unwrap()); // May destroy *this. Copy.Map->Map.erase(Copy); // Definitely destroys *this. } @@ -279,9 +274,9 @@ public: // Make a copy that won't get changed even when *this is destroyed. ValueMapCallbackVH Copy(*this); typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data); - unique_lock<typename Config::mutex_type> Guard; + std::unique_lock<typename Config::mutex_type> Guard; if (M) - Guard = unique_lock<typename Config::mutex_type>(*M); + Guard = std::unique_lock<typename Config::mutex_type>(*M); KeyT typed_new_key = cast<KeySansPointerT>(new_key); // Can destroy *this: diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index 164d0be2855a..49f69340c828 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -132,6 +132,7 @@ void initializeDwarfEHPreparePass(PassRegistry&); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); void initializeEarlyIfConverterPass(PassRegistry&); +void initializeEarlyIfPredicatorPass(PassRegistry &); void initializeEarlyMachineLICMPass(PassRegistry&); void initializeEarlyTailDuplicatePass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); @@ -202,6 +203,7 @@ void initializeLegacyLICMPassPass(PassRegistry&); void initializeLegacyLoopSinkPassPass(PassRegistry&); void initializeLegalizerPass(PassRegistry&); void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &); +void initializeGISelKnownBitsAnalysisPass(PassRegistry &); void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&); void initializeLintPass(PassRegistry&); void initializeLiveDebugValuesPass(PassRegistry&); @@ -241,6 +243,7 @@ void initializeLoopVectorizePass(PassRegistry&); void initializeLoopVersioningLICMPass(PassRegistry&); void initializeLoopVersioningPassPass(PassRegistry&); void initializeLowerAtomicLegacyPassPass(PassRegistry&); +void initializeLowerConstantIntrinsicsPass(PassRegistry&); void initializeLowerEmuTLSPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&); @@ -250,6 +253,7 @@ void initializeLowerInvokeLegacyPassPass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); void initializeLowerTypeTestsPass(PassRegistry&); void initializeMIRCanonicalizerPass(PassRegistry &); +void initializeMIRNamerPass(PassRegistry &); void initializeMIRPrintingPassPass(PassRegistry&); void initializeMachineBlockFrequencyInfoPass(PassRegistry&); void initializeMachineBlockPlacementPass(PassRegistry&); @@ -263,7 +267,7 @@ void initializeMachineDominatorTreePass(PassRegistry&); void initializeMachineFunctionPrinterPassPass(PassRegistry&); void initializeMachineLICMPass(PassRegistry&); void initializeMachineLoopInfoPass(PassRegistry&); -void initializeMachineModuleInfoPass(PassRegistry&); +void initializeMachineModuleInfoWrapperPassPass(PassRegistry &); void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&); void initializeMachineOutlinerPass(PassRegistry&); void initializeMachinePipelinerPass(PassRegistry&); @@ -286,7 +290,9 @@ void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&); +void initializeModuloScheduleTestPass(PassRegistry&); void initializeMustExecutePrinterPass(PassRegistry&); +void initializeMustBeExecutedContextPrinterPass(PassRegistry&); void initializeNameAnonGlobalLegacyPassPass(PassRegistry&); void initializeNaryReassociateLegacyPassPass(PassRegistry&); void initializeNewGVNLegacyPassPass(PassRegistry&); @@ -360,7 +366,7 @@ void initializeSROALegacyPassPass(PassRegistry&); void initializeSafeStackLegacyPassPass(PassRegistry&); void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&); -void initializeSanitizerCoverageModulePass(PassRegistry&); +void initializeModuleSanitizerCoverageLegacyPassPass(PassRegistry &); void initializeScalarEvolutionWrapperPassPass(PassRegistry&); void initializeScalarizeMaskedMemIntrinPass(PassRegistry&); void initializeScalarizerLegacyPassPass(PassRegistry&); diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h index fb107e3fbe02..daa6585b1113 100644 --- a/include/llvm/LTO/Config.h +++ b/include/llvm/LTO/Config.h @@ -226,7 +226,7 @@ struct LTOLLVMContext : LLVMContext { setDiscardValueNames(C.ShouldDiscardValueNames); enableDebugTypeODRUniquing(); setDiagnosticHandler( - llvm::make_unique<LTOLLVMDiagnosticHandler>(&DiagHandler), true); + std::make_unique<LTOLLVMDiagnosticHandler>(&DiagHandler), true); } DiagnosticHandlerFunction DiagHandler; }; diff --git a/include/llvm/LTO/LTO.h b/include/llvm/LTO/LTO.h index ca0a8b64523a..0a1e3e1d0e42 100644 --- a/include/llvm/LTO/LTO.h +++ b/include/llvm/LTO/LTO.h @@ -59,7 +59,9 @@ void thinLTOResolvePrevailingInIndex( /// must apply the changes to the Module via thinLTOInternalizeModule. void thinLTOInternalizeAndPromoteInIndex( ModuleSummaryIndex &Index, - function_ref<bool(StringRef, GlobalValue::GUID)> isExported); + function_ref<bool(StringRef, GlobalValue::GUID)> isExported, + function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> + isPrevailing); /// Computes a unique hash for the Module considering the current list of /// export/import and other global analysis results. @@ -296,6 +298,10 @@ public: /// Cache) for each task identifier. Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr); + /// Static method that returns a list of libcall symbols that can be generated + /// by LTO but might not be visible from bitcode symbol table. + static ArrayRef<const char*> getRuntimeLibcallSymbols(); + private: Config Conf; @@ -303,7 +309,7 @@ private: RegularLTOState(unsigned ParallelCodeGenParallelismLevel, Config &Conf); struct CommonResolution { uint64_t Size = 0; - unsigned Align = 0; + MaybeAlign Align; /// Record if at least one instance of the common was marked as prevailing bool Prevailing = false; }; diff --git a/include/llvm/LTO/legacy/LTOCodeGenerator.h b/include/llvm/LTO/legacy/LTOCodeGenerator.h index d3cb4c8b79a0..8718df4b88e6 100644 --- a/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -113,7 +113,7 @@ struct LTOCodeGenerator { ShouldRestoreGlobalsLinkage = Value; } - void addMustPreserveSymbol(StringRef Sym) { MustPreserveSymbols[Sym] = 1; } + void addMustPreserveSymbol(StringRef Sym) { MustPreserveSymbols.insert(Sym); } /// Pass options to the driver and optimization passes. /// diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h index 675d179eb22a..ac88165845d3 100644 --- a/include/llvm/LinkAllPasses.h +++ b/include/llvm/LinkAllPasses.h @@ -140,6 +140,7 @@ namespace { (void) llvm::createLoopVersioningLICMPass(); (void) llvm::createLoopIdiomPass(); (void) llvm::createLoopRotatePass(); + (void) llvm::createLowerConstantIntrinsicsPass(); (void) llvm::createLowerExpectIntrinsicPass(); (void) llvm::createLowerInvokePass(); (void) llvm::createLowerSwitchPass(); @@ -219,6 +220,7 @@ namespace { (void) llvm::createStraightLineStrengthReducePass(); (void) llvm::createMemDerefPrinter(); (void) llvm::createMustExecutePrinter(); + (void) llvm::createMustBeExecutedContextPrinter(); (void) llvm::createFloat2IntPass(); (void) llvm::createEliminateAvailableExternallyPass(); (void) llvm::createScalarizeMaskedMemIntrinPass(); diff --git a/include/llvm/MC/MCAsmInfo.h b/include/llvm/MC/MCAsmInfo.h index 971e9354da8c..3261c483e0d8 100644 --- a/include/llvm/MC/MCAsmInfo.h +++ b/include/llvm/MC/MCAsmInfo.h @@ -165,6 +165,10 @@ protected: /// instead. bool UseDataRegionDirectives = false; + /// True if .align is to be used for alignment. Only power-of-two + /// alignment is supported. + bool UseDotAlignForAlignment = false; + //===--- Data Emission Directives -------------------------------------===// /// This should be set to the directive used to get some number of zero bytes @@ -313,6 +317,10 @@ protected: /// Defaults to false. bool HasLinkOnceDirective = false; + /// True if we have a .lglobl directive, which is used to emit the information + /// of a static symbol into the symbol table. Defaults to false. + bool HasDotLGloblDirective = false; + /// This attribute, if not MCSA_Invalid, is used to declare a symbol as having /// hidden visibility. Defaults to MCSA_Hidden. MCSymbolAttr HiddenVisibilityAttr = MCSA_Hidden; @@ -388,6 +396,9 @@ protected: // %hi(), and similar unary operators. bool HasMipsExpressions = false; + // If true, emit function descriptor symbol on AIX. + bool NeedsFunctionDescriptors = false; + public: explicit MCAsmInfo(); virtual ~MCAsmInfo(); @@ -520,6 +531,10 @@ public: return UseDataRegionDirectives; } + bool useDotAlignForAlignment() const { + return UseDotAlignForAlignment; + } + const char *getZeroDirective() const { return ZeroDirective; } const char *getAsciiDirective() const { return AsciiDirective; } const char *getAscizDirective() const { return AscizDirective; } @@ -557,6 +572,8 @@ public: bool hasLinkOnceDirective() const { return HasLinkOnceDirective; } + bool hasDotLGloblDirective() const { return HasDotLGloblDirective; } + MCSymbolAttr getHiddenVisibilityAttr() const { return HiddenVisibilityAttr; } MCSymbolAttr getHiddenDeclarationVisibilityAttr() const { @@ -639,6 +656,7 @@ public: bool canRelaxRelocations() const { return RelaxELFRelocations; } void setRelaxELFRelocations(bool V) { RelaxELFRelocations = V; } bool hasMipsExpressions() const { return HasMipsExpressions; } + bool needsFunctionDescriptors() const { return NeedsFunctionDescriptors; } }; } // end namespace llvm diff --git a/include/llvm/MC/MCAsmInfoXCOFF.h b/include/llvm/MC/MCAsmInfoXCOFF.h index 2a72ba7398a7..4a3bacc954e0 100644 --- a/include/llvm/MC/MCAsmInfoXCOFF.h +++ b/include/llvm/MC/MCAsmInfoXCOFF.h @@ -18,6 +18,11 @@ class MCAsmInfoXCOFF : public MCAsmInfo { protected: MCAsmInfoXCOFF(); + +public: + // Return true only when the identifier Name does not need quotes to be + // syntactically correct for XCOFF. + bool isValidUnquotedName(StringRef Name) const override; }; } // end namespace llvm diff --git a/include/llvm/MC/MCAsmMacro.h b/include/llvm/MC/MCAsmMacro.h index 364d3b5f3666..7eecce0faf64 100644 --- a/include/llvm/MC/MCAsmMacro.h +++ b/include/llvm/MC/MCAsmMacro.h @@ -124,7 +124,6 @@ public: } void dump(raw_ostream &OS) const; - void dump() const { dump(dbgs()); } }; struct MCAsmMacroParameter { @@ -133,10 +132,10 @@ struct MCAsmMacroParameter { bool Required = false; bool Vararg = false; - MCAsmMacroParameter() = default; - +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void dump() const { dump(dbgs()); } - void dump(raw_ostream &OS) const; + LLVM_DUMP_METHOD void dump(raw_ostream &OS) const; +#endif }; typedef std::vector<MCAsmMacroParameter> MCAsmMacroParameters; @@ -149,8 +148,10 @@ public: MCAsmMacro(StringRef N, StringRef B, MCAsmMacroParameters P) : Name(N), Body(B), Parameters(std::move(P)) {} +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void dump() const { dump(dbgs()); } - void dump(raw_ostream &OS) const; + LLVM_DUMP_METHOD void dump(raw_ostream &OS) const; +#endif }; } // namespace llvm diff --git a/include/llvm/MC/MCContext.h b/include/llvm/MC/MCContext.h index 5c2124cc0d15..b925f3218883 100644 --- a/include/llvm/MC/MCContext.h +++ b/include/llvm/MC/MCContext.h @@ -22,6 +22,7 @@ #include "llvm/MC/MCAsmMacro.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" @@ -112,6 +113,9 @@ namespace llvm { /// number of section symbols with the same name). StringMap<bool, BumpPtrAllocator &> UsedNames; + /// Keeps track of labels that are used in inline assembly. + SymbolTable InlineAsmUsedLabelNames; + /// The next ID to dole out to an unnamed assembler temporary symbol with /// a given prefix. StringMap<unsigned> NextID; @@ -275,6 +279,8 @@ namespace llvm { /// Do automatic reset in destructor bool AutoReset; + MCTargetOptions const *TargetOptions; + bool HadError = false; MCSymbol *createSymbolImpl(const StringMapEntry<bool> *Name, @@ -298,7 +304,9 @@ namespace llvm { public: explicit MCContext(const MCAsmInfo *MAI, const MCRegisterInfo *MRI, const MCObjectFileInfo *MOFI, - const SourceMgr *Mgr = nullptr, bool DoAutoReset = true); + const SourceMgr *Mgr = nullptr, + MCTargetOptions const *TargetOpts = nullptr, + bool DoAutoReset = true); MCContext(const MCContext &) = delete; MCContext &operator=(const MCContext &) = delete; ~MCContext(); @@ -377,6 +385,16 @@ namespace llvm { /// APIs. const SymbolTable &getSymbols() const { return Symbols; } + /// isInlineAsmLabel - Return true if the name is a label referenced in + /// inline assembly. + MCSymbol *getInlineAsmLabel(StringRef Name) const { + return InlineAsmUsedLabelNames.lookup(Name); + } + + /// registerInlineAsmLabel - Records that the name is a label referenced in + /// inline assembly. + void registerInlineAsmLabel(MCSymbol *Sym); + /// @} /// \name Section Management @@ -490,6 +508,8 @@ namespace llvm { MCSectionXCOFF *getXCOFFSection(StringRef Section, XCOFF::StorageMappingClass MappingClass, + XCOFF::SymbolType CSectType, + XCOFF::StorageClass StorageClass, SectionKind K, const char *BeginSymName = nullptr); @@ -659,6 +679,7 @@ namespace llvm { bool hadError() { return HadError; } void reportError(SMLoc L, const Twine &Msg); + void reportWarning(SMLoc L, const Twine &Msg); // Unrecoverable error has occurred. Display the best diagnostic we can // and bail via exit(1). For now, most MC backend errors are unrecoverable. // FIXME: We should really do something about that. diff --git a/include/llvm/MC/MCDirectives.h b/include/llvm/MC/MCDirectives.h index 4029264c2026..ea79e68674e5 100644 --- a/include/llvm/MC/MCDirectives.h +++ b/include/llvm/MC/MCDirectives.h @@ -28,6 +28,7 @@ enum MCSymbolAttr { MCSA_ELF_TypeNoType, ///< .type _foo, STT_NOTYPE # aka @notype MCSA_ELF_TypeGnuUniqueObject, /// .type _foo, @gnu_unique_object MCSA_Global, ///< .globl + MCSA_LGlobal, ///< .lglobl (XCOFF) MCSA_Hidden, ///< .hidden (ELF) MCSA_IndirectSymbol, ///< .indirect_symbol (MachO) MCSA_Internal, ///< .internal (ELF) diff --git a/include/llvm/MC/MCDwarf.h b/include/llvm/MC/MCDwarf.h index 1a37aafd0654..a33b4b31bb06 100644 --- a/include/llvm/MC/MCDwarf.h +++ b/include/llvm/MC/MCDwarf.h @@ -629,7 +629,8 @@ public: static void Emit(MCObjectStreamer &streamer, MCAsmBackend *MAB, bool isEH); static void EmitAdvanceLoc(MCObjectStreamer &Streamer, uint64_t AddrDelta); static void EncodeAdvanceLoc(MCContext &Context, uint64_t AddrDelta, - raw_ostream &OS); + raw_ostream &OS, uint32_t *Offset = nullptr, + uint32_t *Size = nullptr); }; } // end namespace llvm diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h index fb23c0114c76..eb2786501f84 100644 --- a/include/llvm/MC/MCExpr.h +++ b/include/llvm/MC/MCExpr.h @@ -48,10 +48,6 @@ private: bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, - const SectionAddrMap *Addrs) const; - - bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, - const MCAsmLayout *Layout, const SectionAddrMap *Addrs, bool InSet) const; protected: @@ -136,7 +132,7 @@ class MCConstantExpr : public MCExpr { int64_t Value; bool PrintInHex = false; - MCConstantExpr(int64_t Value) + explicit MCConstantExpr(int64_t Value) : MCExpr(MCExpr::Constant, SMLoc()), Value(Value) {} MCConstantExpr(int64_t Value, bool PrintInHex) @@ -239,6 +235,8 @@ public: VK_PPC_TOC_LO, // symbol@toc@l VK_PPC_TOC_HI, // symbol@toc@h VK_PPC_TOC_HA, // symbol@toc@ha + VK_PPC_U, // symbol@u + VK_PPC_L, // symbol@l VK_PPC_DTPMOD, // symbol@dtpmod VK_PPC_TPREL_LO, // symbol@tprel@l VK_PPC_TPREL_HI, // symbol@tprel@h diff --git a/include/llvm/MC/MCFixup.h b/include/llvm/MC/MCFixup.h index accffb7f2247..29e321e2354c 100644 --- a/include/llvm/MC/MCFixup.h +++ b/include/llvm/MC/MCFixup.h @@ -20,35 +20,38 @@ class MCExpr; /// Extensible enumeration to represent the type of a fixup. enum MCFixupKind { - FK_NONE = 0, ///< A no-op fixup. - FK_Data_1, ///< A one-byte fixup. - FK_Data_2, ///< A two-byte fixup. - FK_Data_4, ///< A four-byte fixup. - FK_Data_8, ///< A eight-byte fixup. - FK_PCRel_1, ///< A one-byte pc relative fixup. - FK_PCRel_2, ///< A two-byte pc relative fixup. - FK_PCRel_4, ///< A four-byte pc relative fixup. - FK_PCRel_8, ///< A eight-byte pc relative fixup. - FK_GPRel_1, ///< A one-byte gp relative fixup. - FK_GPRel_2, ///< A two-byte gp relative fixup. - FK_GPRel_4, ///< A four-byte gp relative fixup. - FK_GPRel_8, ///< A eight-byte gp relative fixup. - FK_DTPRel_4, ///< A four-byte dtp relative fixup. - FK_DTPRel_8, ///< A eight-byte dtp relative fixup. - FK_TPRel_4, ///< A four-byte tp relative fixup. - FK_TPRel_8, ///< A eight-byte tp relative fixup. - FK_SecRel_1, ///< A one-byte section relative fixup. - FK_SecRel_2, ///< A two-byte section relative fixup. - FK_SecRel_4, ///< A four-byte section relative fixup. - FK_SecRel_8, ///< A eight-byte section relative fixup. - FK_Data_Add_1, ///< A one-byte add fixup. - FK_Data_Add_2, ///< A two-byte add fixup. - FK_Data_Add_4, ///< A four-byte add fixup. - FK_Data_Add_8, ///< A eight-byte add fixup. - FK_Data_Sub_1, ///< A one-byte sub fixup. - FK_Data_Sub_2, ///< A two-byte sub fixup. - FK_Data_Sub_4, ///< A four-byte sub fixup. - FK_Data_Sub_8, ///< A eight-byte sub fixup. + FK_NONE = 0, ///< A no-op fixup. + FK_Data_1, ///< A one-byte fixup. + FK_Data_2, ///< A two-byte fixup. + FK_Data_4, ///< A four-byte fixup. + FK_Data_8, ///< A eight-byte fixup. + FK_Data_6b, ///< A six-bits fixup. + FK_PCRel_1, ///< A one-byte pc relative fixup. + FK_PCRel_2, ///< A two-byte pc relative fixup. + FK_PCRel_4, ///< A four-byte pc relative fixup. + FK_PCRel_8, ///< A eight-byte pc relative fixup. + FK_GPRel_1, ///< A one-byte gp relative fixup. + FK_GPRel_2, ///< A two-byte gp relative fixup. + FK_GPRel_4, ///< A four-byte gp relative fixup. + FK_GPRel_8, ///< A eight-byte gp relative fixup. + FK_DTPRel_4, ///< A four-byte dtp relative fixup. + FK_DTPRel_8, ///< A eight-byte dtp relative fixup. + FK_TPRel_4, ///< A four-byte tp relative fixup. + FK_TPRel_8, ///< A eight-byte tp relative fixup. + FK_SecRel_1, ///< A one-byte section relative fixup. + FK_SecRel_2, ///< A two-byte section relative fixup. + FK_SecRel_4, ///< A four-byte section relative fixup. + FK_SecRel_8, ///< A eight-byte section relative fixup. + FK_Data_Add_1, ///< A one-byte add fixup. + FK_Data_Add_2, ///< A two-byte add fixup. + FK_Data_Add_4, ///< A four-byte add fixup. + FK_Data_Add_8, ///< A eight-byte add fixup. + FK_Data_Add_6b, ///< A six-bits add fixup. + FK_Data_Sub_1, ///< A one-byte sub fixup. + FK_Data_Sub_2, ///< A two-byte sub fixup. + FK_Data_Sub_4, ///< A four-byte sub fixup. + FK_Data_Sub_8, ///< A eight-byte sub fixup. + FK_Data_Sub_6b, ///< A six-bits sub fixup. FirstTargetFixupKind = 128, @@ -75,25 +78,25 @@ class MCFixup { /// The value to put into the fixup location. The exact interpretation of the /// expression is target dependent, usually it will be one of the operands to /// an instruction or an assembler directive. - const MCExpr *Value; + const MCExpr *Value = nullptr; /// The byte index of start of the relocation inside the MCFragment. - uint32_t Offset; + uint32_t Offset = 0; /// The target dependent kind of fixup item this is. The kind is used to /// determine how the operand value should be encoded into the instruction. - unsigned Kind; + MCFixupKind Kind = FK_NONE; /// The source location which gave rise to the fixup, if any. SMLoc Loc; public: static MCFixup create(uint32_t Offset, const MCExpr *Value, MCFixupKind Kind, SMLoc Loc = SMLoc()) { - assert(unsigned(Kind) < MaxTargetFixupKind && "Kind out of range!"); + assert(Kind < MaxTargetFixupKind && "Kind out of range!"); MCFixup FI; FI.Value = Value; FI.Offset = Offset; - FI.Kind = unsigned(Kind); + FI.Kind = Kind; FI.Loc = Loc; return FI; } @@ -104,7 +107,7 @@ public: MCFixup FI; FI.Value = Fixup.getValue(); FI.Offset = Fixup.getOffset(); - FI.Kind = (unsigned)getAddKindForKind(Fixup.getKind()); + FI.Kind = getAddKindForKind(Fixup.getKind()); FI.Loc = Fixup.getLoc(); return FI; } @@ -115,12 +118,14 @@ public: MCFixup FI; FI.Value = Fixup.getValue(); FI.Offset = Fixup.getOffset(); - FI.Kind = (unsigned)getSubKindForKind(Fixup.getKind()); + FI.Kind = getSubKindForKind(Fixup.getKind()); FI.Loc = Fixup.getLoc(); return FI; } - MCFixupKind getKind() const { return MCFixupKind(Kind); } + MCFixupKind getKind() const { return Kind; } + + unsigned getTargetKind() const { return Kind; } uint32_t getOffset() const { return Offset; } void setOffset(uint32_t Value) { Offset = Value; } @@ -129,37 +134,63 @@ public: /// Return the generic fixup kind for a value with the given size. It /// is an error to pass an unsupported size. - static MCFixupKind getKindForSize(unsigned Size, bool isPCRel) { + static MCFixupKind getKindForSize(unsigned Size, bool IsPCRel) { switch (Size) { default: llvm_unreachable("Invalid generic fixup size!"); - case 1: return isPCRel ? FK_PCRel_1 : FK_Data_1; - case 2: return isPCRel ? FK_PCRel_2 : FK_Data_2; - case 4: return isPCRel ? FK_PCRel_4 : FK_Data_4; - case 8: return isPCRel ? FK_PCRel_8 : FK_Data_8; + case 1: + return IsPCRel ? FK_PCRel_1 : FK_Data_1; + case 2: + return IsPCRel ? FK_PCRel_2 : FK_Data_2; + case 4: + return IsPCRel ? FK_PCRel_4 : FK_Data_4; + case 8: + return IsPCRel ? FK_PCRel_8 : FK_Data_8; + } + } + + /// Return the generic fixup kind for a value with the given size in bits. + /// It is an error to pass an unsupported size. + static MCFixupKind getKindForSizeInBits(unsigned Size, bool IsPCRel) { + switch (Size) { + default: + llvm_unreachable("Invalid generic fixup size!"); + case 6: + assert(!IsPCRel && "Invalid pc-relative fixup size!"); + return FK_Data_6b; + case 8: + return IsPCRel ? FK_PCRel_1 : FK_Data_1; + case 16: + return IsPCRel ? FK_PCRel_2 : FK_Data_2; + case 32: + return IsPCRel ? FK_PCRel_4 : FK_Data_4; + case 64: + return IsPCRel ? FK_PCRel_8 : FK_Data_8; } } /// Return the generic fixup kind for an addition with a given size. It /// is an error to pass an unsupported size. - static MCFixupKind getAddKindForKind(unsigned Kind) { + static MCFixupKind getAddKindForKind(MCFixupKind Kind) { switch (Kind) { default: llvm_unreachable("Unknown type to convert!"); case FK_Data_1: return FK_Data_Add_1; case FK_Data_2: return FK_Data_Add_2; case FK_Data_4: return FK_Data_Add_4; case FK_Data_8: return FK_Data_Add_8; + case FK_Data_6b: return FK_Data_Add_6b; } } /// Return the generic fixup kind for an subtraction with a given size. It /// is an error to pass an unsupported size. - static MCFixupKind getSubKindForKind(unsigned Kind) { + static MCFixupKind getSubKindForKind(MCFixupKind Kind) { switch (Kind) { default: llvm_unreachable("Unknown type to convert!"); case FK_Data_1: return FK_Data_Sub_1; case FK_Data_2: return FK_Data_Sub_2; case FK_Data_4: return FK_Data_Sub_4; case FK_Data_8: return FK_Data_Sub_8; + case FK_Data_6b: return FK_Data_Sub_6b; } } diff --git a/include/llvm/MC/MCFragment.h b/include/llvm/MC/MCFragment.h index aadf2ce725ea..b0def566c46a 100644 --- a/include/llvm/MC/MCFragment.h +++ b/include/llvm/MC/MCFragment.h @@ -149,6 +149,7 @@ public: case MCFragment::FT_CompactEncodedInst: case MCFragment::FT_Data: case MCFragment::FT_Dwarf: + case MCFragment::FT_DwarfFrame: return true; } } @@ -232,7 +233,8 @@ public: static bool classof(const MCFragment *F) { MCFragment::FragmentType Kind = F->getKind(); return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data || - Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf;; + Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf || + Kind == MCFragment::FT_DwarfFrame; } }; @@ -543,27 +545,21 @@ public: } }; -class MCDwarfCallFrameFragment : public MCFragment { +class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> { /// AddrDelta - The expression for the difference of the two symbols that /// make up the address delta between two .cfi_* dwarf directives. const MCExpr *AddrDelta; - SmallString<8> Contents; - public: MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSection *Sec = nullptr) - : MCFragment(FT_DwarfFrame, false, Sec), AddrDelta(&AddrDelta) { - Contents.push_back(0); - } + : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false, Sec), + AddrDelta(&AddrDelta) {} /// \name Accessors /// @{ const MCExpr &getAddrDelta() const { return *AddrDelta; } - SmallString<8> &getContents() { return Contents; } - const SmallString<8> &getContents() const { return Contents; } - /// @} static bool classof(const MCFragment *F) { diff --git a/include/llvm/MC/MCInstPrinter.h b/include/llvm/MC/MCInstPrinter.h index 6bbc4bc2903b..4501ce3084c8 100644 --- a/include/llvm/MC/MCInstPrinter.h +++ b/include/llvm/MC/MCInstPrinter.h @@ -87,12 +87,10 @@ public: /// Utility functions to make adding mark ups simpler. StringRef markup(StringRef s) const; - StringRef markup(StringRef a, StringRef b) const; bool getPrintImmHex() const { return PrintImmHex; } void setPrintImmHex(bool Value) { PrintImmHex = Value; } - HexStyle::Style getPrintHexStyle() const { return PrintHexStyle; } void setPrintHexStyle(HexStyle::Style Value) { PrintHexStyle = Value; } /// Utility function to print immediates in decimal or hex. diff --git a/include/llvm/MC/MCInstrAnalysis.h b/include/llvm/MC/MCInstrAnalysis.h index dfefd7e72777..898ca47b13b8 100644 --- a/include/llvm/MC/MCInstrAnalysis.h +++ b/include/llvm/MC/MCInstrAnalysis.h @@ -152,6 +152,12 @@ public: evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, uint64_t &Target) const; + /// Given an instruction tries to get the address of a memory operand. Returns + /// the address on success. + virtual Optional<uint64_t> evaluateMemoryOperandAddress(const MCInst &Inst, + uint64_t Addr, + uint64_t Size) const; + /// Returns (PLT virtual address, GOT virtual address) pairs for PLT entries. virtual std::vector<std::pair<uint64_t, uint64_t>> findPltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents, diff --git a/include/llvm/MC/MCInstrDesc.h b/include/llvm/MC/MCInstrDesc.h index 0aa586dfc901..e75a27614a22 100644 --- a/include/llvm/MC/MCInstrDesc.h +++ b/include/llvm/MC/MCInstrDesc.h @@ -56,7 +56,11 @@ enum OperandType { OPERAND_GENERIC_5 = 11, OPERAND_LAST_GENERIC = 11, - OPERAND_FIRST_TARGET = 12, + OPERAND_FIRST_GENERIC_IMM = 12, + OPERAND_GENERIC_IMM_0 = 12, + OPERAND_LAST_GENERIC_IMM = 12, + + OPERAND_FIRST_TARGET = 13, }; } @@ -103,6 +107,16 @@ public: assert(isGenericType() && "non-generic types don't have an index"); return OperandType - MCOI::OPERAND_FIRST_GENERIC; } + + bool isGenericImm() const { + return OperandType >= MCOI::OPERAND_FIRST_GENERIC_IMM && + OperandType <= MCOI::OPERAND_LAST_GENERIC_IMM; + } + + unsigned getGenericImmIndex() const { + assert(isGenericImm() && "non-generic immediates don't have an index"); + return OperandType - MCOI::OPERAND_FIRST_GENERIC_IMM; + } }; //===----------------------------------------------------------------------===// @@ -115,7 +129,8 @@ namespace MCID { /// not use these directly. These all correspond to bitfields in the /// MCInstrDesc::Flags field. enum Flag { - Variadic = 0, + PreISelOpcode = 0, + Variadic, HasOptionalDef, Pseudo, Return, @@ -228,6 +243,10 @@ public: /// Return flags of this instruction. uint64_t getFlags() const { return Flags; } + /// \returns true if this instruction is emitted before instruction selection + /// and should be legalized/regbankselected/selected. + bool isPreISelOpcode() const { return Flags & (1ULL << MCID::PreISelOpcode); } + /// Return true if this instruction can have a variable number of /// operands. In this case, the variable operands will be after the normal /// operands but before the implicit definitions and uses (if any are diff --git a/include/llvm/MC/MCLinkerOptimizationHint.h b/include/llvm/MC/MCLinkerOptimizationHint.h index f2a1364ad884..003491f32f75 100644 --- a/include/llvm/MC/MCLinkerOptimizationHint.h +++ b/include/llvm/MC/MCLinkerOptimizationHint.h @@ -61,6 +61,7 @@ static inline int MCLOHNameToId(StringRef Name) { MCLOHCaseNameToId(AdrpAdd) MCLOHCaseNameToId(AdrpLdrGot) .Default(-1); +#undef MCLOHCaseNameToId } static inline StringRef MCLOHIdToName(MCLOHType Kind) { @@ -76,6 +77,7 @@ static inline StringRef MCLOHIdToName(MCLOHType Kind) { MCLOHCaseIdToName(AdrpLdrGot); } return StringRef(); +#undef MCLOHCaseIdToName } static inline int MCLOHIdToNbArgs(MCLOHType Kind) { diff --git a/include/llvm/MC/MCRegister.h b/include/llvm/MC/MCRegister.h new file mode 100644 index 000000000000..8372947a4ba1 --- /dev/null +++ b/include/llvm/MC/MCRegister.h @@ -0,0 +1,110 @@ +//===-- llvm/MC/Register.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_REGISTER_H +#define LLVM_MC_REGISTER_H + +#include "llvm/ADT/DenseMapInfo.h" +#include <cassert> + +namespace llvm { + +/// An unsigned integer type large enough to represent all physical registers, +/// but not necessarily virtual registers. +using MCPhysReg = uint16_t; + +/// Wrapper class representing physical registers. Should be passed by value. +class MCRegister { + unsigned Reg; + +public: + MCRegister(unsigned Val = 0): Reg(Val) {} + + // Register numbers can represent physical registers, virtual registers, and + // sometimes stack slots. The unsigned values are divided into these ranges: + // + // 0 Not a register, can be used as a sentinel. + // [1;2^30) Physical registers assigned by TableGen. + // [2^30;2^31) Stack slots. (Rarely used.) + // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. + // + // Further sentinels can be allocated from the small negative integers. + // DenseMapInfo<unsigned> uses -1u and -2u. + + /// This is the portion of the positive number space that is not a physical + /// register. StackSlot values do not exist in the MC layer, see + /// Register::isStackSlot() for the more information on them. + /// + /// Note that isVirtualRegister() and isPhysicalRegister() cannot handle stack + /// slots, so if a variable may contains a stack slot, always check + /// isStackSlot() first. + static bool isStackSlot(unsigned Reg) { + return int(Reg) >= (1 << 30); + } + + /// Return true if the specified register number is in + /// the physical register namespace. + static bool isPhysicalRegister(unsigned Reg) { + assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); + return int(Reg) > 0; + } + + /// Return true if the specified register number is in the physical register + /// namespace. + bool isPhysical() const { + return isPhysicalRegister(Reg); + } + + operator unsigned() const { + return Reg; + } + + unsigned id() const { + return Reg; + } + + bool isValid() const { + return Reg != 0; + } + + /// Comparisons between register objects + bool operator==(const MCRegister &Other) const { return Reg == Other.Reg; } + bool operator!=(const MCRegister &Other) const { return Reg != Other.Reg; } + + /// Comparisons against register constants. E.g. + /// * R == AArch64::WZR + /// * R == 0 + /// * R == VirtRegMap::NO_PHYS_REG + bool operator==(unsigned Other) const { return Reg == Other; } + bool operator!=(unsigned Other) const { return Reg != Other; } + bool operator==(int Other) const { return Reg == unsigned(Other); } + bool operator!=(int Other) const { return Reg != unsigned(Other); } + // MSVC requires that we explicitly declare these two as well. + bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } + bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } +}; + +// Provide DenseMapInfo for MCRegister +template<> struct DenseMapInfo<MCRegister> { + static inline unsigned getEmptyKey() { + return DenseMapInfo<unsigned>::getEmptyKey(); + } + static inline unsigned getTombstoneKey() { + return DenseMapInfo<unsigned>::getTombstoneKey(); + } + static unsigned getHashValue(const MCRegister &Val) { + return DenseMapInfo<unsigned>::getHashValue(Val.id()); + } + static bool isEqual(const MCRegister &LHS, const MCRegister &RHS) { + return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); + } +}; + +} + +#endif // ifndef LLVM_MC_REGISTER_H diff --git a/include/llvm/MC/MCRegisterInfo.h b/include/llvm/MC/MCRegisterInfo.h index 92d39c3fcfb7..c7dc56ea588e 100644 --- a/include/llvm/MC/MCRegisterInfo.h +++ b/include/llvm/MC/MCRegisterInfo.h @@ -18,16 +18,13 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/iterator_range.h" #include "llvm/MC/LaneBitmask.h" +#include "llvm/MC/MCRegister.h" #include <cassert> #include <cstdint> #include <utility> namespace llvm { -/// An unsigned integer type large enough to represent all physical registers, -/// but not necessarily virtual registers. -using MCPhysReg = uint16_t; - /// MCRegisterClass - Base class of TargetRegisterClass. class MCRegisterClass { public: @@ -65,16 +62,17 @@ public: /// contains - Return true if the specified register is included in this /// register class. This does not include virtual registers. - bool contains(unsigned Reg) const { - unsigned InByte = Reg % 8; - unsigned Byte = Reg / 8; + bool contains(MCRegister Reg) const { + unsigned RegNo = unsigned(Reg); + unsigned InByte = RegNo % 8; + unsigned Byte = RegNo / 8; if (Byte >= RegSetSize) return false; return (RegSet[Byte] & (1 << InByte)) != 0; } /// contains - Return true if both registers are in this class. - bool contains(unsigned Reg1, unsigned Reg2) const { + bool contains(MCRegister Reg1, MCRegister Reg2) const { return contains(Reg1) && contains(Reg2); } @@ -148,8 +146,8 @@ public: private: const MCRegisterDesc *Desc; // Pointer to the descriptor array unsigned NumRegs; // Number of entries in the array - unsigned RAReg; // Return address register - unsigned PCReg; // Program counter register + MCRegister RAReg; // Return address register + MCRegister PCReg; // Program counter register const MCRegisterClass *Classes; // Pointer to the regclass array unsigned NumClasses; // Number of entries in the array unsigned NumRegUnits; // Number of regunits. @@ -175,8 +173,8 @@ private: const DwarfLLVMRegPair *EHL2DwarfRegs; // LLVM to Dwarf regs mapping EH const DwarfLLVMRegPair *Dwarf2LRegs; // Dwarf to LLVM regs mapping const DwarfLLVMRegPair *EHDwarf2LRegs; // Dwarf to LLVM regs mapping EH - DenseMap<unsigned, int> L2SEHRegs; // LLVM to SEH regs mapping - DenseMap<unsigned, int> L2CVRegs; // LLVM to CV regs mapping + DenseMap<MCRegister, int> L2SEHRegs; // LLVM to SEH regs mapping + DenseMap<MCRegister, int> L2CVRegs; // LLVM to CV regs mapping public: /// DiffListIterator - Base iterator class that can traverse the @@ -202,7 +200,7 @@ public: /// advance - Move to the next list position, return the applied /// differential. This function does not detect the end of the list, that /// is the caller's responsibility (by checking for a 0 return value). - unsigned advance() { + MCRegister advance() { assert(isValid() && "Cannot move off the end of the list."); MCPhysReg D = *List++; Val += D; @@ -214,7 +212,7 @@ public: bool isValid() const { return List; } /// Dereference the iterator to get the value at the current position. - unsigned operator*() const { return Val; } + MCRegister operator*() const { return Val; } /// Pre-increment to move to the next position. void operator++() { @@ -309,26 +307,26 @@ public: /// as the LLVM register number. /// FIXME: TableGen these numbers. Currently this requires target specific /// initialization code. - void mapLLVMRegToSEHReg(unsigned LLVMReg, int SEHReg) { + void mapLLVMRegToSEHReg(MCRegister LLVMReg, int SEHReg) { L2SEHRegs[LLVMReg] = SEHReg; } - void mapLLVMRegToCVReg(unsigned LLVMReg, int CVReg) { + void mapLLVMRegToCVReg(MCRegister LLVMReg, int CVReg) { L2CVRegs[LLVMReg] = CVReg; } /// This method should return the register where the return /// address can be found. - unsigned getRARegister() const { + MCRegister getRARegister() const { return RAReg; } /// Return the register which is the program counter. - unsigned getProgramCounter() const { + MCRegister getProgramCounter() const { return PCReg; } - const MCRegisterDesc &operator[](unsigned RegNo) const { + const MCRegisterDesc &operator[](MCRegister RegNo) const { assert(RegNo < NumRegs && "Attempting to access record for invalid register number!"); return Desc[RegNo]; @@ -336,24 +334,24 @@ public: /// Provide a get method, equivalent to [], but more useful with a /// pointer to this object. - const MCRegisterDesc &get(unsigned RegNo) const { + const MCRegisterDesc &get(MCRegister RegNo) const { return operator[](RegNo); } /// Returns the physical register number of sub-register "Index" /// for physical register RegNo. Return zero if the sub-register does not /// exist. - unsigned getSubReg(unsigned Reg, unsigned Idx) const; + MCRegister getSubReg(MCRegister Reg, unsigned Idx) const; /// Return a super-register of the specified register /// Reg so its sub-register of index SubIdx is Reg. - unsigned getMatchingSuperReg(unsigned Reg, unsigned SubIdx, - const MCRegisterClass *RC) const; + MCRegister getMatchingSuperReg(MCRegister Reg, unsigned SubIdx, + const MCRegisterClass *RC) const; /// For a given register pair, return the sub-register index /// if the second register is a sub-register of the first. Return zero /// otherwise. - unsigned getSubRegIndex(unsigned RegNo, unsigned SubRegNo) const; + unsigned getSubRegIndex(MCRegister RegNo, MCRegister SubRegNo) const; /// Get the size of the bit range covered by a sub-register index. /// If the index isn't continuous, return the sum of the sizes of its parts. @@ -367,7 +365,7 @@ public: /// Return the human-readable symbolic target-specific name for the /// specified physical register. - const char *getName(unsigned RegNo) const { + const char *getName(MCRegister RegNo) const { return RegStrings + get(RegNo).Name; } @@ -395,15 +393,11 @@ public: /// number. Returns -1 if there is no equivalent value. The second /// parameter allows targets to use different numberings for EH info and /// debugging info. - int getDwarfRegNum(unsigned RegNum, bool isEH) const; - - /// Map a dwarf register back to a target register. - int getLLVMRegNum(unsigned RegNum, bool isEH) const; + int getDwarfRegNum(MCRegister RegNum, bool isEH) const; - /// Map a DWARF EH register back to a target register (same as - /// getLLVMRegNum(RegNum, true)) but return -1 if there is no mapping, - /// rather than asserting that there must be one. - int getLLVMRegNumFromEH(unsigned RegNum) const; + /// Map a dwarf register back to a target register. Returns None is there is + /// no mapping. + Optional<unsigned> getLLVMRegNum(unsigned RegNum, bool isEH) const; /// Map a target EH register number to an equivalent DWARF register /// number. @@ -411,11 +405,11 @@ public: /// Map a target register to an equivalent SEH register /// number. Returns LLVM register number if there is no equivalent value. - int getSEHRegNum(unsigned RegNum) const; + int getSEHRegNum(MCRegister RegNum) const; /// Map a target register to an equivalent CodeView register /// number. - int getCodeViewRegNum(unsigned RegNum) const; + int getCodeViewRegNum(MCRegister RegNum) const; regclass_iterator regclass_begin() const { return Classes; } regclass_iterator regclass_end() const { return Classes+NumClasses; } @@ -439,34 +433,34 @@ public: } /// Returns the encoding for RegNo - uint16_t getEncodingValue(unsigned RegNo) const { + uint16_t getEncodingValue(MCRegister RegNo) const { assert(RegNo < NumRegs && "Attempting to get encoding for invalid register number!"); return RegEncodingTable[RegNo]; } /// Returns true if RegB is a sub-register of RegA. - bool isSubRegister(unsigned RegA, unsigned RegB) const { + bool isSubRegister(MCRegister RegA, MCRegister RegB) const { return isSuperRegister(RegB, RegA); } /// Returns true if RegB is a super-register of RegA. - bool isSuperRegister(unsigned RegA, unsigned RegB) const; + bool isSuperRegister(MCRegister RegA, MCRegister RegB) const; /// Returns true if RegB is a sub-register of RegA or if RegB == RegA. - bool isSubRegisterEq(unsigned RegA, unsigned RegB) const { + bool isSubRegisterEq(MCRegister RegA, MCRegister RegB) const { return isSuperRegisterEq(RegB, RegA); } /// Returns true if RegB is a super-register of RegA or if /// RegB == RegA. - bool isSuperRegisterEq(unsigned RegA, unsigned RegB) const { + bool isSuperRegisterEq(MCRegister RegA, MCRegister RegB) const { return RegA == RegB || isSuperRegister(RegA, RegB); } /// Returns true if RegB is a super-register or sub-register of RegA /// or if RegB == RegA. - bool isSuperOrSubRegisterEq(unsigned RegA, unsigned RegB) const { + bool isSuperOrSubRegisterEq(MCRegister RegA, MCRegister RegB) const { return isSubRegisterEq(RegA, RegB) || isSuperRegister(RegA, RegB); } }; @@ -482,8 +476,8 @@ public: /// If IncludeSelf is set, Reg itself is included in the list. class MCSubRegIterator : public MCRegisterInfo::DiffListIterator { public: - MCSubRegIterator(unsigned Reg, const MCRegisterInfo *MCRI, - bool IncludeSelf = false) { + MCSubRegIterator(MCRegister Reg, const MCRegisterInfo *MCRI, + bool IncludeSelf = false) { init(Reg, MCRI->DiffLists + MCRI->get(Reg).SubRegs); // Initially, the iterator points to Reg itself. if (!IncludeSelf) @@ -500,13 +494,13 @@ class MCSubRegIndexIterator { public: /// Constructs an iterator that traverses subregisters and their /// associated subregister indices. - MCSubRegIndexIterator(unsigned Reg, const MCRegisterInfo *MCRI) + MCSubRegIndexIterator(MCRegister Reg, const MCRegisterInfo *MCRI) : SRIter(Reg, MCRI) { SRIndex = MCRI->SubRegIndices + MCRI->get(Reg).SubRegIndices; } /// Returns current sub-register. - unsigned getSubReg() const { + MCRegister getSubReg() const { return *SRIter; } @@ -531,7 +525,7 @@ class MCSuperRegIterator : public MCRegisterInfo::DiffListIterator { public: MCSuperRegIterator() = default; - MCSuperRegIterator(unsigned Reg, const MCRegisterInfo *MCRI, + MCSuperRegIterator(MCRegister Reg, const MCRegisterInfo *MCRI, bool IncludeSelf = false) { init(Reg, MCRI->DiffLists + MCRI->get(Reg).SuperRegs); // Initially, the iterator points to Reg itself. @@ -542,7 +536,7 @@ public: // Definition for isSuperRegister. Put it down here since it needs the // iterator defined above in addition to the MCRegisterInfo class itself. -inline bool MCRegisterInfo::isSuperRegister(unsigned RegA, unsigned RegB) const{ +inline bool MCRegisterInfo::isSuperRegister(MCRegister RegA, MCRegister RegB) const{ for (MCSuperRegIterator I(RegA, this); I.isValid(); ++I) if (*I == RegB) return true; @@ -569,7 +563,7 @@ public: /// in Reg. MCRegUnitIterator() = default; - MCRegUnitIterator(unsigned Reg, const MCRegisterInfo *MCRI) { + MCRegUnitIterator(MCRegister Reg, const MCRegisterInfo *MCRI) { assert(Reg && "Null register has no regunits"); // Decode the RegUnits MCRegisterDesc field. unsigned RU = MCRI->get(Reg).RegUnits; @@ -600,7 +594,7 @@ public: /// Constructs an iterator that traverses the register units and their /// associated LaneMasks in Reg. - MCRegUnitMaskIterator(unsigned Reg, const MCRegisterInfo *MCRI) + MCRegUnitMaskIterator(MCRegister Reg, const MCRegisterInfo *MCRI) : RUIter(Reg, MCRI) { uint16_t Idx = MCRI->get(Reg).RegUnitLaneMasks; MaskListIter = &MCRI->RegUnitMaskSequences[Idx]; @@ -667,7 +661,7 @@ public: /// any ordering or that entries are unique. class MCRegAliasIterator { private: - unsigned Reg; + MCRegister Reg; const MCRegisterInfo *MCRI; bool IncludeSelf; @@ -676,7 +670,7 @@ private: MCSuperRegIterator SI; public: - MCRegAliasIterator(unsigned Reg, const MCRegisterInfo *MCRI, + MCRegAliasIterator(MCRegister Reg, const MCRegisterInfo *MCRI, bool IncludeSelf) : Reg(Reg), MCRI(MCRI), IncludeSelf(IncludeSelf) { // Initialize the iterators. @@ -692,7 +686,7 @@ public: bool isValid() const { return RI.isValid(); } - unsigned operator*() const { + MCRegister operator*() const { assert(SI.isValid() && "Cannot dereference an invalid iterator."); return *SI; } diff --git a/include/llvm/MC/MCSection.h b/include/llvm/MC/MCSection.h index 6fad1ec2069c..d057feda87d8 100644 --- a/include/llvm/MC/MCSection.h +++ b/include/llvm/MC/MCSection.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ilist.h" #include "llvm/MC/MCFragment.h" #include "llvm/MC/SectionKind.h" +#include "llvm/Support/Alignment.h" #include <cassert> #include <utility> @@ -58,7 +59,7 @@ private: MCSymbol *Begin; MCSymbol *End = nullptr; /// The alignment requirement of this section. - unsigned Alignment = 1; + Align Alignment; /// The section index in the assemblers section list. unsigned Ordinal = 0; /// The index of this section in the layout order. @@ -117,8 +118,8 @@ public: MCSymbol *getEndSymbol(MCContext &Ctx); bool hasEnded() const; - unsigned getAlignment() const { return Alignment; } - void setAlignment(unsigned Value) { Alignment = Value; } + unsigned getAlignment() const { return Alignment.value(); } + void setAlignment(Align Value) { Alignment = Value; } unsigned getOrdinal() const { return Ordinal; } void setOrdinal(unsigned Value) { Ordinal = Value; } diff --git a/include/llvm/MC/MCSectionXCOFF.h b/include/llvm/MC/MCSectionXCOFF.h index 2a3f391fd3e2..ee302ed5ecec 100644 --- a/include/llvm/MC/MCSectionXCOFF.h +++ b/include/llvm/MC/MCSectionXCOFF.h @@ -23,16 +23,30 @@ class MCSymbol; // This class represents an XCOFF `Control Section`, more commonly referred to // as a csect. A csect represents the smallest possible unit of data/code which -// will be relocated as a single block. +// will be relocated as a single block. A csect can either be: +// 1) Initialized: The Type will be XTY_SD, and the symbols inside the csect +// will have a label definition representing their offset within the csect. +// 2) Uninitialized: The Type will be XTY_CM, it will contain a single symbol, +// and may not contain label definitions. +// 3) An external reference providing a symbol table entry for a symbol +// contained in another XCOFF object file. External reference csects are not +// implemented yet. class MCSectionXCOFF final : public MCSection { friend class MCContext; StringRef Name; XCOFF::StorageMappingClass MappingClass; + XCOFF::SymbolType Type; + XCOFF::StorageClass StorageClass; MCSectionXCOFF(StringRef Section, XCOFF::StorageMappingClass SMC, - SectionKind K, MCSymbol *Begin) - : MCSection(SV_XCOFF, K, Begin), Name(Section), MappingClass(SMC) {} + XCOFF::SymbolType ST, XCOFF::StorageClass SC, SectionKind K, + MCSymbol *Begin) + : MCSection(SV_XCOFF, K, Begin), Name(Section), MappingClass(SMC), + Type(ST), StorageClass(SC) { + assert((ST == XCOFF::XTY_SD || ST == XCOFF::XTY_CM) && + "Invalid or unhandled type for csect."); + } public: ~MCSectionXCOFF(); @@ -43,6 +57,8 @@ public: StringRef getSectionName() const { return Name; } XCOFF::StorageMappingClass getMappingClass() const { return MappingClass; } + XCOFF::StorageClass getStorageClass() const { return StorageClass; } + XCOFF::SymbolType getCSectType() const { return Type; } void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h index 731e7515448c..6b48580ae57c 100644 --- a/include/llvm/MC/MCStreamer.h +++ b/include/llvm/MC/MCStreamer.h @@ -46,6 +46,7 @@ struct MCDwarfFrameInfo; class MCExpr; class MCInst; class MCInstPrinter; +class MCRegister; class MCSection; class MCStreamer; class MCSymbolRefExpr; @@ -53,6 +54,13 @@ class MCSubtargetInfo; class raw_ostream; class Twine; +namespace codeview { +struct DefRangeRegisterRelHeader; +struct DefRangeSubfieldRegisterHeader; +struct DefRangeRegisterHeader; +struct DefRangeFramePointerRelHeader; +} + using MCSectionSubPair = std::pair<MCSection *, const MCExpr *>; /// Target specific streamer interface. This is used so that targets can @@ -536,6 +544,15 @@ public: /// \param Symbol - Symbol the image relative relocation should point to. virtual void EmitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset); + /// Emits an lcomm directive with XCOFF csect information. + /// + /// \param Symbol - The symbol we are emiting. + /// \param Size - The size of the block of storage. + /// \param ByteAlignment - The alignment of the symbol in bytes. Must be a power + /// of 2. + virtual void EmitXCOFFLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment); + /// Emit an ELF .size directive. /// /// This corresponds to an assembler statement such as: @@ -860,6 +877,22 @@ public: ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges, StringRef FixedSizePortion); + virtual void EmitCVDefRangeDirective( + ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges, + codeview::DefRangeRegisterRelHeader DRHdr); + + virtual void EmitCVDefRangeDirective( + ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges, + codeview::DefRangeSubfieldRegisterHeader DRHdr); + + virtual void EmitCVDefRangeDirective( + ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges, + codeview::DefRangeRegisterHeader DRHdr); + + virtual void EmitCVDefRangeDirective( + ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges, + codeview::DefRangeFramePointerRelHeader DRHdr); + /// This implements the CodeView '.cv_stringtable' assembler directive. virtual void EmitCVStringTableDirective() {} @@ -917,13 +950,13 @@ public: virtual void EmitWinCFIFuncletOrFuncEnd(SMLoc Loc = SMLoc()); virtual void EmitWinCFIStartChained(SMLoc Loc = SMLoc()); virtual void EmitWinCFIEndChained(SMLoc Loc = SMLoc()); - virtual void EmitWinCFIPushReg(unsigned Register, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISetFrame(unsigned Register, unsigned Offset, + virtual void EmitWinCFIPushReg(MCRegister Register, SMLoc Loc = SMLoc()); + virtual void EmitWinCFISetFrame(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); virtual void EmitWinCFIAllocStack(unsigned Size, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISaveReg(unsigned Register, unsigned Offset, + virtual void EmitWinCFISaveReg(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISaveXMM(unsigned Register, unsigned Offset, + virtual void EmitWinCFISaveXMM(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); virtual void EmitWinCFIPushFrame(bool Code, SMLoc Loc = SMLoc()); virtual void EmitWinCFIEndProlog(SMLoc Loc = SMLoc()); diff --git a/include/llvm/MC/MCSubtargetInfo.h b/include/llvm/MC/MCSubtargetInfo.h index 9490a6ecedad..09130c4641ef 100644 --- a/include/llvm/MC/MCSubtargetInfo.h +++ b/include/llvm/MC/MCSubtargetInfo.h @@ -221,6 +221,52 @@ public: auto Found = std::lower_bound(ProcDesc.begin(), ProcDesc.end(), CPU); return Found != ProcDesc.end() && StringRef(Found->Key) == CPU; } + + virtual unsigned getHwMode() const { return 0; } + + /// Return the cache size in bytes for the given level of cache. + /// Level is zero-based, so a value of zero means the first level of + /// cache. + /// + virtual Optional<unsigned> getCacheSize(unsigned Level) const; + + /// Return the cache associatvity for the given level of cache. + /// Level is zero-based, so a value of zero means the first level of + /// cache. + /// + virtual Optional<unsigned> getCacheAssociativity(unsigned Level) const; + + /// Return the target cache line size in bytes at a given level. + /// + virtual Optional<unsigned> getCacheLineSize(unsigned Level) const; + + /// Return the target cache line size in bytes. By default, return + /// the line size for the bottom-most level of cache. This provides + /// a more convenient interface for the common case where all cache + /// levels have the same line size. Return zero if there is no + /// cache model. + /// + virtual unsigned getCacheLineSize() const { + Optional<unsigned> Size = getCacheLineSize(0); + if (Size) + return *Size; + + return 0; + } + + /// Return the preferred prefetch distance in terms of instructions. + /// + virtual unsigned getPrefetchDistance() const; + + /// Return the maximum prefetch distance in terms of loop + /// iterations. + /// + virtual unsigned getMaxPrefetchIterationsAhead() const; + + /// Return the minimum stride necessary to trigger software + /// prefetching. + /// + virtual unsigned getMinPrefetchStride() const; }; } // end namespace llvm diff --git a/include/llvm/MC/MCSymbolWasm.h b/include/llvm/MC/MCSymbolWasm.h index c50cd0ee4709..95beebe3f75a 100644 --- a/include/llvm/MC/MCSymbolWasm.h +++ b/include/llvm/MC/MCSymbolWasm.h @@ -54,6 +54,13 @@ public: modifyFlags(wasm::WASM_SYMBOL_EXPORTED, wasm::WASM_SYMBOL_EXPORTED); } + bool isNoStrip() const { + return getFlags() & wasm::WASM_SYMBOL_NO_STRIP; + } + void setNoStrip() const { + modifyFlags(wasm::WASM_SYMBOL_NO_STRIP, wasm::WASM_SYMBOL_NO_STRIP); + } + bool isWeak() const { return IsWeak; } void setWeak(bool isWeak) { IsWeak = isWeak; } diff --git a/include/llvm/MC/MCSymbolXCOFF.h b/include/llvm/MC/MCSymbolXCOFF.h index 0a1fe1475138..98ecd2466926 100644 --- a/include/llvm/MC/MCSymbolXCOFF.h +++ b/include/llvm/MC/MCSymbolXCOFF.h @@ -8,17 +8,49 @@ #ifndef LLVM_MC_MCSYMBOLXCOFF_H #define LLVM_MC_MCSYMBOLXCOFF_H +#include "llvm/ADT/Optional.h" #include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCSymbol.h" namespace llvm { +class MCSectionXCOFF; + class MCSymbolXCOFF : public MCSymbol { public: MCSymbolXCOFF(const StringMapEntry<bool> *Name, bool isTemporary) : MCSymbol(SymbolKindXCOFF, Name, isTemporary) {} static bool classof(const MCSymbol *S) { return S->isXCOFF(); } + + void setStorageClass(XCOFF::StorageClass SC) { + assert((!StorageClass.hasValue() || StorageClass.getValue() == SC) && + "Redefining StorageClass of XCOFF MCSymbol."); + StorageClass = SC; + }; + + XCOFF::StorageClass getStorageClass() const { + assert(StorageClass.hasValue() && + "StorageClass not set on XCOFF MCSymbol."); + return StorageClass.getValue(); + } + + void setContainingCsect(MCSectionXCOFF *C) { + assert((!ContainingCsect || ContainingCsect == C) && + "Trying to set a containing csect that doesn't match the one that" + "this symbol is already mapped to."); + ContainingCsect = C; + } + + MCSectionXCOFF *getContainingCsect() const { + assert(ContainingCsect && + "Trying to get containing csect but none was set."); + return ContainingCsect; + } + +private: + Optional<XCOFF::StorageClass> StorageClass; + MCSectionXCOFF *ContainingCsect = nullptr; }; } // end namespace llvm diff --git a/include/llvm/MC/MCWasmObjectWriter.h b/include/llvm/MC/MCWasmObjectWriter.h index 4adbca28f116..fbb68549b503 100644 --- a/include/llvm/MC/MCWasmObjectWriter.h +++ b/include/llvm/MC/MCWasmObjectWriter.h @@ -20,9 +20,10 @@ class raw_pwrite_stream; class MCWasmObjectTargetWriter : public MCObjectTargetWriter { const unsigned Is64Bit : 1; + const unsigned IsEmscripten : 1; protected: - explicit MCWasmObjectTargetWriter(bool Is64Bit_); + explicit MCWasmObjectTargetWriter(bool Is64Bit_, bool IsEmscripten); public: virtual ~MCWasmObjectTargetWriter(); @@ -38,6 +39,7 @@ public: /// \name Accessors /// @{ bool is64Bit() const { return Is64Bit; } + bool isEmscripten() const { return IsEmscripten; } /// @} }; diff --git a/include/llvm/MC/MCXCOFFStreamer.h b/include/llvm/MC/MCXCOFFStreamer.h index 159ae4818749..b13b0031d18e 100644 --- a/include/llvm/MC/MCXCOFFStreamer.h +++ b/include/llvm/MC/MCXCOFFStreamer.h @@ -26,6 +26,8 @@ public: uint64_t Size = 0, unsigned ByteAlignment = 0, SMLoc Loc = SMLoc()) override; void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override; + void EmitXCOFFLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlign) override; }; } // end namespace llvm diff --git a/include/llvm/MC/StringTableBuilder.h b/include/llvm/MC/StringTableBuilder.h index c83eca4e512d..c8d4c3bbc262 100644 --- a/include/llvm/MC/StringTableBuilder.h +++ b/include/llvm/MC/StringTableBuilder.h @@ -22,7 +22,7 @@ class raw_ostream; /// Utility for building string tables with deduplicated suffixes. class StringTableBuilder { public: - enum Kind { ELF, WinCOFF, MachO, RAW, DWARF }; + enum Kind { ELF, WinCOFF, MachO, RAW, DWARF, XCOFF }; private: DenseMap<CachedHashStringRef, size_t> StringIndexMap; diff --git a/include/llvm/MC/SubtargetFeature.h b/include/llvm/MC/SubtargetFeature.h index fc9565ceafad..defbc3c64720 100644 --- a/include/llvm/MC/SubtargetFeature.h +++ b/include/llvm/MC/SubtargetFeature.h @@ -18,6 +18,7 @@ #define LLVM_MC_SUBTARGETFEATURE_H #include "llvm/ADT/StringRef.h" +#include "llvm/Support/MathExtras.h" #include <array> #include <bitset> #include <initializer_list> @@ -33,20 +34,123 @@ const unsigned MAX_SUBTARGET_WORDS = 3; const unsigned MAX_SUBTARGET_FEATURES = MAX_SUBTARGET_WORDS * 64; /// Container class for subtarget features. -/// This is convenient because std::bitset does not have a constructor -/// with an initializer list of set bits. -class FeatureBitset : public std::bitset<MAX_SUBTARGET_FEATURES> { -public: - // Cannot inherit constructors because it's not supported by VC++.. - FeatureBitset() = default; - - FeatureBitset(const bitset<MAX_SUBTARGET_FEATURES>& B) : bitset(B) {} +/// This is a constexpr reimplementation of a subset of std::bitset. It would be +/// nice to use std::bitset directly, but it doesn't support constant +/// initialization. +class FeatureBitset { + static_assert((MAX_SUBTARGET_FEATURES % 64) == 0, + "Should be a multiple of 64!"); + // This cannot be a std::array, operator[] is not constexpr until C++17. + uint64_t Bits[MAX_SUBTARGET_WORDS] = {}; + +protected: + constexpr FeatureBitset(const std::array<uint64_t, MAX_SUBTARGET_WORDS> &B) { + for (unsigned I = 0; I != B.size(); ++I) + Bits[I] = B[I]; + } - FeatureBitset(std::initializer_list<unsigned> Init) { +public: + constexpr FeatureBitset() = default; + constexpr FeatureBitset(std::initializer_list<unsigned> Init) { for (auto I : Init) set(I); } + FeatureBitset &set() { + std::fill(std::begin(Bits), std::end(Bits), -1ULL); + return *this; + } + + constexpr FeatureBitset &set(unsigned I) { + // GCC <6.2 crashes if this is written in a single statement. + uint64_t NewBits = Bits[I / 64] | (uint64_t(1) << (I % 64)); + Bits[I / 64] = NewBits; + return *this; + } + + constexpr FeatureBitset &reset(unsigned I) { + // GCC <6.2 crashes if this is written in a single statement. + uint64_t NewBits = Bits[I / 64] & ~(uint64_t(1) << (I % 64)); + Bits[I / 64] = NewBits; + return *this; + } + + constexpr FeatureBitset &flip(unsigned I) { + // GCC <6.2 crashes if this is written in a single statement. + uint64_t NewBits = Bits[I / 64] ^ (uint64_t(1) << (I % 64)); + Bits[I / 64] = NewBits; + return *this; + } + + constexpr bool operator[](unsigned I) const { + uint64_t Mask = uint64_t(1) << (I % 64); + return (Bits[I / 64] & Mask) != 0; + } + + constexpr bool test(unsigned I) const { return (*this)[I]; } + + constexpr size_t size() const { return MAX_SUBTARGET_FEATURES; } + + bool any() const { + return llvm::any_of(Bits, [](uint64_t I) { return I != 0; }); + } + bool none() const { return !any(); } + size_t count() const { + size_t Count = 0; + for (auto B : Bits) + Count += countPopulation(B); + return Count; + } + + constexpr FeatureBitset &operator^=(const FeatureBitset &RHS) { + for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I) { + Bits[I] ^= RHS.Bits[I]; + } + return *this; + } + constexpr FeatureBitset operator^(const FeatureBitset &RHS) const { + FeatureBitset Result = *this; + Result ^= RHS; + return Result; + } + + constexpr FeatureBitset &operator&=(const FeatureBitset &RHS) { + for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I) { + Bits[I] &= RHS.Bits[I]; + } + return *this; + } + constexpr FeatureBitset operator&(const FeatureBitset &RHS) const { + FeatureBitset Result = *this; + Result &= RHS; + return Result; + } + + constexpr FeatureBitset &operator|=(const FeatureBitset &RHS) { + for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I) { + Bits[I] |= RHS.Bits[I]; + } + return *this; + } + constexpr FeatureBitset operator|(const FeatureBitset &RHS) const { + FeatureBitset Result = *this; + Result |= RHS; + return Result; + } + + constexpr FeatureBitset operator~() const { + FeatureBitset Result = *this; + for (auto &B : Result.Bits) + B = ~B; + return Result; + } + + bool operator==(const FeatureBitset &RHS) const { + return std::equal(std::begin(Bits), std::end(Bits), std::begin(RHS.Bits)); + } + + bool operator!=(const FeatureBitset &RHS) const { return !(*this == RHS); } + bool operator < (const FeatureBitset &Other) const { for (unsigned I = 0, E = size(); I != E; ++I) { bool LHS = test(I), RHS = Other.test(I); @@ -58,23 +162,12 @@ public: }; /// Class used to store the subtarget bits in the tables created by tablegen. -/// The std::initializer_list constructor of FeatureBitset can't be done at -/// compile time and requires a static constructor to run at startup. -class FeatureBitArray { - std::array<uint64_t, MAX_SUBTARGET_WORDS> Bits; - +class FeatureBitArray : public FeatureBitset { public: constexpr FeatureBitArray(const std::array<uint64_t, MAX_SUBTARGET_WORDS> &B) - : Bits(B) {} - - FeatureBitset getAsBitset() const { - FeatureBitset Result; - - for (unsigned i = 0, e = Bits.size(); i != e; ++i) - Result |= FeatureBitset(Bits[i]) << (64 * i); + : FeatureBitset(B) {} - return Result; - } + const FeatureBitset &getAsBitset() const { return *this; } }; //===----------------------------------------------------------------------===// diff --git a/include/llvm/MCA/CodeEmitter.h b/include/llvm/MCA/CodeEmitter.h new file mode 100644 index 000000000000..c8d222bd8c2f --- /dev/null +++ b/include/llvm/MCA/CodeEmitter.h @@ -0,0 +1,72 @@ +//===--------------------- CodeEmitter.h ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// A utility class used to compute instruction encodings. It buffers encodings +/// for later usage. It exposes a simple API to compute and get the encodings as +/// StringRef. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MCA_CODEEMITTER_H +#define LLVM_MCA_CODEEMITTER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MCA/Instruction.h" +#include "llvm/MCA/Support.h" +#include "llvm/Support/raw_ostream.h" + +#include <string> + +namespace llvm { +namespace mca { + +/// A utility class used to compute instruction encodings for a code region. +/// +/// It provides a simple API to compute and return instruction encodings as +/// strings. Encodings are cached internally for later usage. +class CodeEmitter { + const MCSubtargetInfo &STI; + const MCAsmBackend &MAB; + const MCCodeEmitter &MCE; + + SmallString<256> Code; + raw_svector_ostream VecOS; + ArrayRef<MCInst> Sequence; + + // An EncodingInfo pair stores <base, length> information. Base (i.e. first) + // is an index to the `Code`. Length (i.e. second) is the encoding size. + using EncodingInfo = std::pair<unsigned, unsigned>; + + // A cache of encodings. + SmallVector<EncodingInfo, 16> Encodings; + + EncodingInfo getOrCreateEncodingInfo(unsigned MCID); + +public: + CodeEmitter(const MCSubtargetInfo &ST, const MCAsmBackend &AB, + const MCCodeEmitter &CE, ArrayRef<MCInst> S) + : STI(ST), MAB(AB), MCE(CE), VecOS(Code), Sequence(S), + Encodings(S.size()) {} + + StringRef getEncoding(unsigned MCID) { + EncodingInfo EI = getOrCreateEncodingInfo(MCID); + return StringRef(&Code[EI.first], EI.second); + } +}; + +} // namespace mca +} // namespace llvm + +#endif // LLVM_MCA_CODEEMITTER_H diff --git a/include/llvm/MCA/Context.h b/include/llvm/MCA/Context.h index 503d780d4947..af3cb8e1e837 100644 --- a/include/llvm/MCA/Context.h +++ b/include/llvm/MCA/Context.h @@ -20,7 +20,6 @@ #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MCA/HardwareUnits/HardwareUnit.h" -#include "llvm/MCA/InstrBuilder.h" #include "llvm/MCA/Pipeline.h" #include "llvm/MCA/SourceMgr.h" #include <memory> @@ -58,6 +57,9 @@ public: Context(const Context &C) = delete; Context &operator=(const Context &C) = delete; + const MCRegisterInfo &getMCRegisterInfo() const { return MRI; } + const MCSubtargetInfo &getMCSubtargetInfo() const { return STI; } + void addHardwareUnit(std::unique_ptr<HardwareUnit> H) { Hardware.push_back(std::move(H)); } @@ -65,7 +67,6 @@ public: /// Construct a basic pipeline for simulating an out-of-order pipeline. /// This pipeline consists of Fetch, Dispatch, Execute, and Retire stages. std::unique_ptr<Pipeline> createDefaultPipeline(const PipelineOptions &Opts, - InstrBuilder &IB, SourceMgr &SrcMgr); }; diff --git a/include/llvm/MCA/HardwareUnits/LSUnit.h b/include/llvm/MCA/HardwareUnits/LSUnit.h index ae9a49c64855..34903794db4a 100644 --- a/include/llvm/MCA/HardwareUnits/LSUnit.h +++ b/include/llvm/MCA/HardwareUnits/LSUnit.h @@ -209,8 +209,10 @@ public: unsigned getUsedLQEntries() const { return UsedLQEntries; } unsigned getUsedSQEntries() const { return UsedSQEntries; } - unsigned assignLQSlot() { return UsedLQEntries++; } - unsigned assignSQSlot() { return UsedSQEntries++; } + void acquireLQSlot() { ++UsedLQEntries; } + void acquireSQSlot() { ++UsedSQEntries; } + void releaseLQSlot() { --UsedLQEntries; } + void releaseSQSlot() { --UsedSQEntries; } bool assumeNoAlias() const { return NoAlias; } @@ -285,13 +287,18 @@ public: unsigned createMemoryGroup() { Groups.insert( - std::make_pair(NextGroupID, llvm::make_unique<MemoryGroup>())); + std::make_pair(NextGroupID, std::make_unique<MemoryGroup>())); return NextGroupID++; } - // Instruction executed event handlers. virtual void onInstructionExecuted(const InstRef &IR); + // Loads are tracked by the LDQ (load queue) from dispatch until completion. + // Stores are tracked by the STQ (store queue) from dispatch until commitment. + // By default we conservatively assume that the LDQ receives a load at + // dispatch. Loads leave the LDQ at retirement stage. + virtual void onInstructionRetired(const InstRef &IR); + virtual void onInstructionIssued(const InstRef &IR) { unsigned GroupID = IR.getInstruction()->getLSUTokenID(); Groups[GroupID]->onInstructionIssued(IR); @@ -436,9 +443,6 @@ public: /// 6. A store has to wait until an older store barrier is fully executed. unsigned dispatch(const InstRef &IR) override; - // FIXME: For simplicity, we optimistically assume a similar behavior for - // store instructions. In practice, store operations don't tend to leave the - // store queue until they reach the 'Retired' stage (See PR39830). void onInstructionExecuted(const InstRef &IR) override; }; diff --git a/include/llvm/MCA/HardwareUnits/RegisterFile.h b/include/llvm/MCA/HardwareUnits/RegisterFile.h index 36506327bd29..cd7718d98744 100644 --- a/include/llvm/MCA/HardwareUnits/RegisterFile.h +++ b/include/llvm/MCA/HardwareUnits/RegisterFile.h @@ -220,7 +220,7 @@ public: // // Current implementation can simulate up to 32 register files (including the // special register file at index #0). - unsigned isAvailable(ArrayRef<unsigned> Regs) const; + unsigned isAvailable(ArrayRef<MCPhysReg> Regs) const; // Returns the number of PRFs implemented by this processor. unsigned getNumRegisterFiles() const { return RegisterFiles.size(); } diff --git a/include/llvm/MCA/HardwareUnits/ResourceManager.h b/include/llvm/MCA/HardwareUnits/ResourceManager.h index 2f91185516fb..917af3750044 100644 --- a/include/llvm/MCA/HardwareUnits/ResourceManager.h +++ b/include/llvm/MCA/HardwareUnits/ResourceManager.h @@ -33,8 +33,7 @@ namespace mca { /// with a buffer size of -1 is always available if it is not reserved. /// /// Values of type ResourceStateEvent are returned by method -/// ResourceState::isBufferAvailable(), which is used to query the internal -/// state of a resource. +/// ResourceManager::canBeDispatched() /// /// The naming convention for resource state events is: /// * Event names start with prefix RS_ @@ -263,16 +262,26 @@ public: /// Returns RS_BUFFER_UNAVAILABLE if there are no available slots. ResourceStateEvent isBufferAvailable() const; - /// Reserve a slot in the buffer. - void reserveBuffer() { - if (AvailableSlots) - AvailableSlots--; + /// Reserve a buffer slot. + /// + /// Returns true if the buffer is not full. + /// It always returns true if BufferSize is set to zero. + bool reserveBuffer() { + if (BufferSize <= 0) + return true; + + --AvailableSlots; + assert(AvailableSlots <= static_cast<unsigned>(BufferSize)); + return AvailableSlots; } - /// Release a slot in the buffer. + /// Releases a slot in the buffer. void releaseBuffer() { - if (BufferSize > 0) - AvailableSlots++; + // Ignore dispatch hazards or invalid buffer sizes. + if (BufferSize <= 0) + return; + + ++AvailableSlots; assert(AvailableSlots <= static_cast<unsigned>(BufferSize)); } @@ -351,9 +360,16 @@ class ResourceManager { // Set of processor resource units that are available during this cycle. uint64_t AvailableProcResUnits; - // Set of processor resource groups that are currently reserved. + // Set of processor resources that are currently reserved. uint64_t ReservedResourceGroups; + // Set of unavailable scheduler buffer resources. This is used internally to + // speedup `canBeDispatched()` queries. + uint64_t AvailableBuffers; + + // Set of dispatch hazard buffer resources that are currently unavailable. + uint64_t ReservedBuffers; + // Returns the actual resource unit that will be used. ResourceRef selectPipe(uint64_t ResourceID); @@ -382,17 +398,20 @@ public: // Returns RS_BUFFER_AVAILABLE if buffered resources are not reserved, and if // there are enough available slots in the buffers. - ResourceStateEvent canBeDispatched(ArrayRef<uint64_t> Buffers) const; + ResourceStateEvent canBeDispatched(uint64_t ConsumedBuffers) const; // Return the processor resource identifier associated to this Mask. unsigned resolveResourceMask(uint64_t Mask) const; - // Consume a slot in every buffered resource from array 'Buffers'. Resource - // units that are dispatch hazards (i.e. BufferSize=0) are marked as reserved. - void reserveBuffers(ArrayRef<uint64_t> Buffers); + // Acquires a slot from every buffered resource in mask `ConsumedBuffers`. + // Units that are dispatch hazards (i.e. BufferSize=0) are marked as reserved. + void reserveBuffers(uint64_t ConsumedBuffers); - // Release buffer entries previously allocated by method reserveBuffers. - void releaseBuffers(ArrayRef<uint64_t> Buffers); + // Releases a slot from every buffered resource in mask `ConsumedBuffers`. + // ConsumedBuffers is a bitmask of previously acquired buffers (using method + // `reserveBuffers`). Units that are dispatch hazards (i.e. BufferSize=0) are + // not automatically unreserved by this method. + void releaseBuffers(uint64_t ConsumedBuffers); // Reserve a processor resource. A reserved resource is not available for // instruction issue until it is released. diff --git a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h index 06290141739e..acbd4543bd4a 100644 --- a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h +++ b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h @@ -57,34 +57,43 @@ struct RetireControlUnit : public HardwareUnit { private: unsigned NextAvailableSlotIdx; unsigned CurrentInstructionSlotIdx; - unsigned AvailableSlots; + unsigned NumROBEntries; + unsigned AvailableEntries; unsigned MaxRetirePerCycle; // 0 means no limit. std::vector<RUToken> Queue; -public: - RetireControlUnit(const MCSchedModel &SM); - - bool isEmpty() const { return AvailableSlots == Queue.size(); } - bool isAvailable(unsigned Quantity = 1) const { + unsigned normalizeQuantity(unsigned Quantity) const { // Some instructions may declare a number of uOps which exceeds the size // of the reorder buffer. To avoid problems, cap the amount of slots to // the size of the reorder buffer. - Quantity = std::min(Quantity, static_cast<unsigned>(Queue.size())); + Quantity = std::min(Quantity, NumROBEntries); // Further normalize the number of micro opcodes for instructions that // declare zero opcodes. This should match the behavior of method // reserveSlot(). - Quantity = std::max(Quantity, 1U); - return AvailableSlots >= Quantity; + return std::max(Quantity, 1U); + } + + unsigned computeNextSlotIdx() const; + +public: + RetireControlUnit(const MCSchedModel &SM); + + bool isEmpty() const { return AvailableEntries == NumROBEntries; } + + bool isAvailable(unsigned Quantity = 1) const { + return AvailableEntries >= normalizeQuantity(Quantity); } unsigned getMaxRetirePerCycle() const { return MaxRetirePerCycle; } - // Reserves a number of slots, and returns a new token. - unsigned reserveSlot(const InstRef &IS, unsigned NumMicroOps); + // Reserves a number of slots, and returns a new token reference. + unsigned dispatch(const InstRef &IS); // Return the current token from the RCU's circular token queue. - const RUToken &peekCurrentToken() const; + const RUToken &getCurrentToken() const; + + const RUToken &peekNextToken() const; // Advance the pointer to the next token in the circular token queue. void consumeCurrentToken(); diff --git a/include/llvm/MCA/HardwareUnits/Scheduler.h b/include/llvm/MCA/HardwareUnits/Scheduler.h index 27beb842dfd2..6c196757e571 100644 --- a/include/llvm/MCA/HardwareUnits/Scheduler.h +++ b/include/llvm/MCA/HardwareUnits/Scheduler.h @@ -68,7 +68,7 @@ public: /// instructions from the dispatch stage, until the write-back stage. /// class Scheduler : public HardwareUnit { - LSUnit &LSU; + LSUnitBase &LSU; // Instruction selection strategy for this Scheduler. std::unique_ptr<SchedulerStrategy> Strategy; @@ -154,15 +154,15 @@ class Scheduler : public HardwareUnit { bool promoteToPendingSet(SmallVectorImpl<InstRef> &Pending); public: - Scheduler(const MCSchedModel &Model, LSUnit &Lsu) + Scheduler(const MCSchedModel &Model, LSUnitBase &Lsu) : Scheduler(Model, Lsu, nullptr) {} - Scheduler(const MCSchedModel &Model, LSUnit &Lsu, + Scheduler(const MCSchedModel &Model, LSUnitBase &Lsu, std::unique_ptr<SchedulerStrategy> SelectStrategy) - : Scheduler(make_unique<ResourceManager>(Model), Lsu, + : Scheduler(std::make_unique<ResourceManager>(Model), Lsu, std::move(SelectStrategy)) {} - Scheduler(std::unique_ptr<ResourceManager> RM, LSUnit &Lsu, + Scheduler(std::unique_ptr<ResourceManager> RM, LSUnitBase &Lsu, std::unique_ptr<SchedulerStrategy> SelectStrategy) : LSU(Lsu), Resources(std::move(RM)), BusyResourceUnits(0), NumDispatchedToThePendingSet(0), HadTokenStall(false) { @@ -228,6 +228,9 @@ public: SmallVectorImpl<InstRef> &Ready); /// Convert a resource mask into a valid llvm processor resource identifier. + /// + /// Only the most significant bit of the Mask is used by this method to + /// identify the processor resource. unsigned getResourceID(uint64_t Mask) const { return Resources->resolveResourceMask(Mask); } diff --git a/include/llvm/MCA/Instruction.h b/include/llvm/MCA/Instruction.h index d4d3f22797f7..c97cb463d0f5 100644 --- a/include/llvm/MCA/Instruction.h +++ b/include/llvm/MCA/Instruction.h @@ -18,6 +18,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCRegister.h" // definition of MCPhysReg. #include "llvm/Support/MathExtras.h" #ifndef NDEBUG @@ -42,7 +43,7 @@ struct WriteDescriptor { unsigned Latency; // This field is set to a value different than zero only if this // is an implicit definition. - unsigned RegisterID; + MCPhysReg RegisterID; // Instruction itineraries would set this field to the SchedClass ID. // Otherwise, it defaults to the WriteResourceID from the MCWriteLatencyEntry // element associated to this write. @@ -70,7 +71,7 @@ struct ReadDescriptor { // uses always come first in the sequence of uses. unsigned UseIndex; // This field is only set if this is an implicit read. - unsigned RegisterID; + MCPhysReg RegisterID; // Scheduling Class Index. It is used to query the scheduling model for the // MCSchedClassDesc object. unsigned SchedClassID; @@ -85,7 +86,7 @@ class ReadState; /// Field RegID is set to the invalid register for memory dependencies. struct CriticalDependency { unsigned IID; - unsigned RegID; + MCPhysReg RegID; unsigned Cycles; }; @@ -106,7 +107,7 @@ class WriteState { // to speedup queries on the register file. // For implicit writes, this field always matches the value of // field RegisterID from WD. - unsigned RegisterID; + MCPhysReg RegisterID; // Physical register file that serves register RegisterID. unsigned PRFID; @@ -146,7 +147,7 @@ class WriteState { SmallVector<std::pair<ReadState *, int>, 4> Users; public: - WriteState(const WriteDescriptor &Desc, unsigned RegID, + WriteState(const WriteDescriptor &Desc, MCPhysReg RegID, bool clearsSuperRegs = false, bool writesZero = false) : WD(&Desc), CyclesLeft(UNKNOWN_CYCLES), RegisterID(RegID), PRFID(0), ClearsSuperRegs(clearsSuperRegs), WritesZero(writesZero), @@ -158,7 +159,7 @@ public: int getCyclesLeft() const { return CyclesLeft; } unsigned getWriteResourceID() const { return WD->SClassOrWriteResourceID; } - unsigned getRegisterID() const { return RegisterID; } + MCPhysReg getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } unsigned getLatency() const { return WD->Latency; } unsigned getDependentWriteCyclesLeft() const { @@ -200,7 +201,7 @@ public: } void setDependentWrite(const WriteState *Other) { DependentWrite = Other; } - void writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles); + void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles); void setWriteZero() { WritesZero = true; } void setEliminated() { assert(Users.empty() && "Write is in an inconsistent state."); @@ -226,7 +227,7 @@ public: class ReadState { const ReadDescriptor *RD; // Physical register identified associated to this read. - unsigned RegisterID; + MCPhysReg RegisterID; // Physical register file that serves register RegisterID. unsigned PRFID; // Number of writes that contribute to the definition of RegisterID. @@ -253,14 +254,14 @@ class ReadState { bool IndependentFromDef; public: - ReadState(const ReadDescriptor &Desc, unsigned RegID) + ReadState(const ReadDescriptor &Desc, MCPhysReg RegID) : RD(&Desc), RegisterID(RegID), PRFID(0), DependentWrites(0), CyclesLeft(UNKNOWN_CYCLES), TotalCycles(0), CRD(), IsReady(true), IsZero(false), IndependentFromDef(false) {} const ReadDescriptor &getDescriptor() const { return *RD; } unsigned getSchedClass() const { return RD->SchedClassID; } - unsigned getRegisterID() const { return RegisterID; } + MCPhysReg getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } const CriticalDependency &getCriticalRegDep() const { return CRD; } @@ -272,7 +273,7 @@ public: void setIndependentFromDef() { IndependentFromDef = true; } void cycleEvent(); - void writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles); + void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles); void setDependentWrites(unsigned Writes) { DependentWrites = Writes; IsReady = !Writes; @@ -352,11 +353,14 @@ struct InstrDesc { // reports the number of "consumed cycles". SmallVector<std::pair<uint64_t, ResourceUsage>, 4> Resources; - // A list of buffered resources consumed by this instruction. - SmallVector<uint64_t, 4> Buffers; + // A bitmask of used hardware buffers. + uint64_t UsedBuffers; - unsigned UsedProcResUnits; - unsigned UsedProcResGroups; + // A bitmask of used processor resource units. + uint64_t UsedProcResUnits; + + // A bitmask of used processor resource groups. + uint64_t UsedProcResGroups; unsigned MaxLatency; // Number of MicroOps for this instruction. @@ -414,6 +418,7 @@ public: const InstrDesc &getDesc() const { return Desc; } unsigned getLatency() const { return Desc.MaxLatency; } + unsigned getNumMicroOps() const { return Desc.NumMicroOps; } bool hasDependentUsers() const { return any_of(Defs, @@ -463,6 +468,12 @@ class Instruction : public InstructionBase { // operation. unsigned LSUTokenID; + // A resource mask which identifies buffered resources consumed by this + // instruction at dispatch stage. In the absence of macro-fusion, this value + // should always match the value of field `UsedBuffers` from the instruction + // descriptor (see field InstrBase::Desc). + uint64_t UsedBuffers; + // Critical register dependency. CriticalDependency CriticalRegDep; @@ -480,12 +491,18 @@ class Instruction : public InstructionBase { public: Instruction(const InstrDesc &D) : InstructionBase(D), Stage(IS_INVALID), CyclesLeft(UNKNOWN_CYCLES), - RCUTokenID(0), LSUTokenID(0), CriticalRegDep(), CriticalMemDep(), - CriticalResourceMask(0), IsEliminated(false) {} + RCUTokenID(0), LSUTokenID(0), UsedBuffers(D.UsedBuffers), + CriticalRegDep(), CriticalMemDep(), CriticalResourceMask(0), + IsEliminated(false) {} unsigned getRCUTokenID() const { return RCUTokenID; } unsigned getLSUTokenID() const { return LSUTokenID; } void setLSUTokenID(unsigned LSUTok) { LSUTokenID = LSUTok; } + + uint64_t getUsedBuffers() const { return UsedBuffers; } + void setUsedBuffers(uint64_t Mask) { UsedBuffers = Mask; } + void clearUsedBuffers() { UsedBuffers = 0ULL; } + int getCyclesLeft() const { return CyclesLeft; } // Transition to the dispatch stage, and assign a RCUToken to this diff --git a/include/llvm/MCA/SourceMgr.h b/include/llvm/MCA/SourceMgr.h index dbe31db1b1dd..e844171bdcab 100644 --- a/include/llvm/MCA/SourceMgr.h +++ b/include/llvm/MCA/SourceMgr.h @@ -16,12 +16,13 @@ #define LLVM_MCA_SOURCEMGR_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/MCA/Instruction.h" namespace llvm { namespace mca { -class Instruction; - +// MSVC >= 19.15, < 19.20 need to see the definition of class Instruction to +// prevent compiler error C2139 about intrinsic type trait '__is_assignable'. typedef std::pair<unsigned, const Instruction &> SourceRef; class SourceMgr { diff --git a/include/llvm/MCA/Stages/RetireStage.h b/include/llvm/MCA/Stages/RetireStage.h index 08c216ac7bf4..f4713688d25f 100644 --- a/include/llvm/MCA/Stages/RetireStage.h +++ b/include/llvm/MCA/Stages/RetireStage.h @@ -16,6 +16,7 @@ #ifndef LLVM_MCA_RETIRE_STAGE_H #define LLVM_MCA_RETIRE_STAGE_H +#include "llvm/MCA/HardwareUnits/LSUnit.h" #include "llvm/MCA/HardwareUnits/RegisterFile.h" #include "llvm/MCA/HardwareUnits/RetireControlUnit.h" #include "llvm/MCA/Stages/Stage.h" @@ -27,13 +28,14 @@ class RetireStage final : public Stage { // Owner will go away when we move listeners/eventing to the stages. RetireControlUnit &RCU; RegisterFile &PRF; + LSUnitBase &LSU; RetireStage(const RetireStage &Other) = delete; RetireStage &operator=(const RetireStage &Other) = delete; public: - RetireStage(RetireControlUnit &R, RegisterFile &F) - : Stage(), RCU(R), PRF(F) {} + RetireStage(RetireControlUnit &R, RegisterFile &F, LSUnitBase &LS) + : Stage(), RCU(R), PRF(F), LSU(LS) {} bool hasWorkToComplete() const override { return !RCU.isEmpty(); } Error cycleStart() override; diff --git a/include/llvm/Object/Archive.h b/include/llvm/Object/Archive.h index c40278a4f923..c3f36bdd9d1a 100644 --- a/include/llvm/Object/Archive.h +++ b/include/llvm/Object/Archive.h @@ -48,8 +48,7 @@ public: /// Get the name looking up long names. Expected<StringRef> getName(uint64_t Size) const; - /// Members are not larger than 4GB. - Expected<uint32_t> getSize() const; + Expected<uint64_t> getSize() const; Expected<sys::fs::perms> getAccessMode() const; Expected<sys::TimePoint<std::chrono::seconds>> getLastModified() const; @@ -136,6 +135,7 @@ public: Expected<StringRef> getBuffer() const; uint64_t getChildOffset() const; + uint64_t getDataOffset() const { return getChildOffset() + StartOfFile; } Expected<MemoryBufferRef> getMemoryBufferRef() const; @@ -221,6 +221,9 @@ public: Archive(MemoryBufferRef Source, Error &Err); static Expected<std::unique_ptr<Archive>> create(MemoryBufferRef Source); + /// Size field is 10 decimal digits long + static const uint64_t MaxMemberSize = 9999999999; + enum Kind { K_GNU, K_GNU64, diff --git a/include/llvm/Object/Binary.h b/include/llvm/Object/Binary.h index 3c3e977baff4..aa5e718f5e9b 100644 --- a/include/llvm/Object/Binary.h +++ b/include/llvm/Object/Binary.h @@ -42,7 +42,9 @@ protected: ID_Archive, ID_MachOUniversalBinary, ID_COFFImportFile, - ID_IR, // LLVM IR + ID_IR, // LLVM IR + ID_TapiUniversal, // Text-based Dynamic Library Stub file. + ID_TapiFile, // Text-based Dynamic Library Stub file. ID_Minidump, @@ -101,16 +103,18 @@ public: return TypeID > ID_StartObjects && TypeID < ID_EndObjects; } - bool isSymbolic() const { return isIR() || isObject() || isCOFFImportFile(); } - - bool isArchive() const { - return TypeID == ID_Archive; + bool isSymbolic() const { + return isIR() || isObject() || isCOFFImportFile() || isTapiFile(); } + bool isArchive() const { return TypeID == ID_Archive; } + bool isMachOUniversalBinary() const { return TypeID == ID_MachOUniversalBinary; } + bool isTapiUniversal() const { return TypeID == ID_TapiUniversal; } + bool isELF() const { return TypeID >= ID_ELF32L && TypeID <= ID_ELF64B; } @@ -137,6 +141,8 @@ public: bool isMinidump() const { return TypeID == ID_Minidump; } + bool isTapiFile() const { return TypeID == ID_TapiFile; } + bool isLittleEndian() const { return !(TypeID == ID_ELF32B || TypeID == ID_ELF64B || TypeID == ID_MachO32B || TypeID == ID_MachO64B); diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h index c53cbc46c747..b91ee5887fec 100644 --- a/include/llvm/Object/COFF.h +++ b/include/llvm/Object/COFF.h @@ -314,7 +314,10 @@ public: return CS16 ? CS16->Name.Offset : CS32->Name.Offset; } - uint32_t getValue() const { return CS16 ? CS16->Value : CS32->Value; } + uint32_t getValue() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + return CS16 ? CS16->Value : CS32->Value; + } int32_t getSectionNumber() const { assert(isSet() && "COFFSymbolRef points to nothing!"); @@ -969,11 +972,14 @@ public: return nullptr; return reinterpret_cast<const dos_header *>(base()); } - std::error_code getCOFFHeader(const coff_file_header *&Res) const; - std::error_code - getCOFFBigObjHeader(const coff_bigobj_file_header *&Res) const; - std::error_code getPE32Header(const pe32_header *&Res) const; - std::error_code getPE32PlusHeader(const pe32plus_header *&Res) const; + + const coff_file_header *getCOFFHeader() const { return COFFHeader; } + const coff_bigobj_file_header *getCOFFBigObjHeader() const { + return COFFBigObjHeader; + } + const pe32_header *getPE32Header() const { return PE32Header; } + const pe32plus_header *getPE32PlusHeader() const { return PE32PlusHeader; } + std::error_code getDataDirectory(uint32_t index, const data_directory *&Res) const; std::error_code getSection(int32_t index, const coff_section *&Res) const; @@ -1201,16 +1207,34 @@ public: ResourceSectionRef() = default; explicit ResourceSectionRef(StringRef Ref) : BBS(Ref, support::little) {} + Error load(const COFFObjectFile *O); + Error load(const COFFObjectFile *O, const SectionRef &S); + Expected<ArrayRef<UTF16>> getEntryNameString(const coff_resource_dir_entry &Entry); Expected<const coff_resource_dir_table &> getEntrySubDir(const coff_resource_dir_entry &Entry); + Expected<const coff_resource_data_entry &> + getEntryData(const coff_resource_dir_entry &Entry); Expected<const coff_resource_dir_table &> getBaseTable(); + Expected<const coff_resource_dir_entry &> + getTableEntry(const coff_resource_dir_table &Table, uint32_t Index); + + Expected<StringRef> getContents(const coff_resource_data_entry &Entry); private: BinaryByteStream BBS; + SectionRef Section; + const COFFObjectFile *Obj; + + std::vector<const coff_relocation *> Relocs; + Expected<const coff_resource_dir_table &> getTableAtOffset(uint32_t Offset); + Expected<const coff_resource_dir_entry &> + getTableEntryAtOffset(uint32_t Offset); + Expected<const coff_resource_data_entry &> + getDataEntryAtOffset(uint32_t Offset); Expected<ArrayRef<UTF16>> getDirStringAtOffset(uint32_t Offset); }; diff --git a/include/llvm/Object/ELF.h b/include/llvm/Object/ELF.h index cf8e4529bad9..28b00c8413de 100644 --- a/include/llvm/Object/ELF.h +++ b/include/llvm/Object/ELF.h @@ -64,6 +64,10 @@ std::string getSecIndexForError(const ELFFile<ELFT> *Obj, return "[unknown index]"; } +static inline Error defaultWarningHandler(const Twine &Msg) { + return createError(Msg); +} + template <class ELFT> class ELFFile { public: @@ -95,6 +99,13 @@ public: using Elf_Relr_Range = typename ELFT::RelrRange; using Elf_Phdr_Range = typename ELFT::PhdrRange; + // This is a callback that can be passed to a number of functions. + // It can be used to ignore non-critical errors (warnings), which is + // useful for dumpers, like llvm-readobj. + // It accepts a warning message string and returns a success + // when the warning should be ignored or an error otherwise. + using WarningHandler = llvm::function_ref<Error(const Twine &Msg)>; + const uint8_t *base() const { return Buf.bytes_begin(); } size_t getBufSize() const { return Buf.size(); } @@ -114,7 +125,9 @@ public: template <typename T> Expected<const T *> getEntry(const Elf_Shdr *Section, uint32_t Entry) const; - Expected<StringRef> getStringTable(const Elf_Shdr *Section) const; + Expected<StringRef> + getStringTable(const Elf_Shdr *Section, + WarningHandler WarnHandler = &defaultWarningHandler) const; Expected<StringRef> getStringTableForSymtab(const Elf_Shdr &Section) const; Expected<StringRef> getStringTableForSymtab(const Elf_Shdr &Section, Elf_Shdr_Range Sections) const; @@ -137,15 +150,16 @@ public: static Expected<ELFFile> create(StringRef Object); + bool isLE() const { + return getHeader()->getDataEncoding() == ELF::ELFDATA2LSB; + } + bool isMipsELF64() const { return getHeader()->e_machine == ELF::EM_MIPS && getHeader()->getFileClass() == ELF::ELFCLASS64; } - bool isMips64EL() const { - return isMipsELF64() && - getHeader()->getDataEncoding() == ELF::ELFDATA2LSB; - } + bool isMips64EL() const { return isMipsELF64() && isLE(); } Expected<Elf_Shdr_Range> sections() const; @@ -261,7 +275,9 @@ public: return make_range(notes_begin(Shdr, Err), notes_end()); } - Expected<StringRef> getSectionStringTable(Elf_Shdr_Range Sections) const; + Expected<StringRef> getSectionStringTable( + Elf_Shdr_Range Sections, + WarningHandler WarnHandler = &defaultWarningHandler) const; Expected<uint32_t> getSectionIndex(const Elf_Sym *Sym, Elf_Sym_Range Syms, ArrayRef<Elf_Word> ShndxTable) const; Expected<const Elf_Shdr *> getSection(const Elf_Sym *Sym, @@ -271,12 +287,13 @@ public: Elf_Sym_Range Symtab, ArrayRef<Elf_Word> ShndxTable) const; Expected<const Elf_Shdr *> getSection(uint32_t Index) const; - Expected<const Elf_Shdr *> getSection(const StringRef SectionName) const; Expected<const Elf_Sym *> getSymbol(const Elf_Shdr *Sec, uint32_t Index) const; - Expected<StringRef> getSectionName(const Elf_Shdr *Section) const; + Expected<StringRef> + getSectionName(const Elf_Shdr *Section, + WarningHandler WarnHandler = &defaultWarningHandler) const; Expected<StringRef> getSectionName(const Elf_Shdr *Section, StringRef DotShstrtab) const; template <typename T> @@ -459,18 +476,18 @@ ELFFile<ELFT>::getRelocationSymbol(const Elf_Rel *Rel, template <class ELFT> Expected<StringRef> -ELFFile<ELFT>::getSectionStringTable(Elf_Shdr_Range Sections) const { +ELFFile<ELFT>::getSectionStringTable(Elf_Shdr_Range Sections, + WarningHandler WarnHandler) const { uint32_t Index = getHeader()->e_shstrndx; if (Index == ELF::SHN_XINDEX) Index = Sections[0].sh_link; if (!Index) // no section string table. return ""; - // TODO: Test a case when the sh_link of the section with index 0 is broken. if (Index >= Sections.size()) return createError("section header string table index " + Twine(Index) + " does not exist"); - return getStringTable(&Sections[Index]); + return getStringTable(&Sections[Index], WarnHandler); } template <class ELFT> ELFFile<ELFT>::ELFFile(StringRef Object) : Buf(Object) {} @@ -495,7 +512,8 @@ Expected<typename ELFT::ShdrRange> ELFFile<ELFT>::sections() const { Twine(getHeader()->e_shentsize)); const uint64_t FileSize = Buf.size(); - if (SectionTableOffset + sizeof(Elf_Shdr) > FileSize) + if (SectionTableOffset + sizeof(Elf_Shdr) > FileSize || + SectionTableOffset + (uintX_t)sizeof(Elf_Shdr) < SectionTableOffset) return createError( "section header table goes past the end of the file: e_shoff = 0x" + Twine::utohexstr(SectionTableOffset)); @@ -513,15 +531,22 @@ Expected<typename ELFT::ShdrRange> ELFFile<ELFT>::sections() const { NumSections = First->sh_size; if (NumSections > UINT64_MAX / sizeof(Elf_Shdr)) - // TODO: this error is untested. - return createError("section table goes past the end of file"); + return createError("invalid number of sections specified in the NULL " + "section's sh_size field (" + + Twine(NumSections) + ")"); const uint64_t SectionTableSize = NumSections * sizeof(Elf_Shdr); + if (SectionTableOffset + SectionTableSize < SectionTableOffset) + return createError( + "invalid section header table offset (e_shoff = 0x" + + Twine::utohexstr(SectionTableOffset) + + ") or invalid number of sections specified in the first section " + "header's sh_size field (0x" + + Twine::utohexstr(NumSections) + ")"); // Section table goes past end of file! if (SectionTableOffset + SectionTableSize > FileSize) return createError("section table goes past the end of file"); - return makeArrayRef(First, NumSections); } @@ -540,8 +565,9 @@ template <typename T> Expected<const T *> ELFFile<ELFT>::getEntry(const Elf_Shdr *Section, uint32_t Entry) const { if (sizeof(T) != Section->sh_entsize) - // TODO: this error is untested. - return createError("invalid sh_entsize"); + return createError("section " + getSecIndexForError(this, Section) + + " has invalid sh_entsize: expected " + Twine(sizeof(T)) + + ", but got " + Twine(Section->sh_entsize)); size_t Pos = Section->sh_offset + Entry * sizeof(T); if (Pos + sizeof(T) > Buf.size()) return createError("unable to access section " + @@ -561,42 +587,26 @@ ELFFile<ELFT>::getSection(uint32_t Index) const { } template <class ELFT> -Expected<const typename ELFT::Shdr *> -ELFFile<ELFT>::getSection(const StringRef SectionName) const { - auto TableOrErr = sections(); - if (!TableOrErr) - return TableOrErr.takeError(); - for (auto &Sec : *TableOrErr) { - auto SecNameOrErr = getSectionName(&Sec); - if (!SecNameOrErr) - return SecNameOrErr.takeError(); - if (*SecNameOrErr == SectionName) - return &Sec; - } - // TODO: this error is untested. - return createError("invalid section name"); -} - -template <class ELFT> Expected<StringRef> -ELFFile<ELFT>::getStringTable(const Elf_Shdr *Section) const { +ELFFile<ELFT>::getStringTable(const Elf_Shdr *Section, + WarningHandler WarnHandler) const { if (Section->sh_type != ELF::SHT_STRTAB) - return createError("invalid sh_type for string table section " + - getSecIndexForError(this, Section) + - ": expected SHT_STRTAB, but got " + - object::getELFSectionTypeName(getHeader()->e_machine, - Section->sh_type)); + if (Error E = WarnHandler("invalid sh_type for string table section " + + getSecIndexForError(this, Section) + + ": expected SHT_STRTAB, but got " + + object::getELFSectionTypeName( + getHeader()->e_machine, Section->sh_type))) + return std::move(E); + auto V = getSectionContentsAsArray<char>(Section); if (!V) return V.takeError(); ArrayRef<char> Data = *V; if (Data.empty()) - // TODO: this error is untested. - return createError("empty string table"); + return createError("SHT_STRTAB string table section " + + getSecIndexForError(this, Section) + " is empty"); if (Data.back() != '\0') - return createError(object::getELFSectionTypeName(getHeader()->e_machine, - Section->sh_type) + - " string table section " + + return createError("SHT_STRTAB string table section " + getSecIndexForError(this, Section) + " is non-null terminated"); return StringRef(Data.begin(), Data.size()); @@ -626,8 +636,11 @@ ELFFile<ELFT>::getSHNDXTable(const Elf_Shdr &Section, const Elf_Shdr &SymTable = **SymTableOrErr; if (SymTable.sh_type != ELF::SHT_SYMTAB && SymTable.sh_type != ELF::SHT_DYNSYM) - // TODO: this error is untested. - return createError("invalid sh_type"); + return createError("SHT_SYMTAB_SHNDX section is linked with " + + object::getELFSectionTypeName(getHeader()->e_machine, + SymTable.sh_type) + + " section (expected SHT_SYMTAB/SHT_DYNSYM)"); + if (V.size() != (SymTable.sh_size / sizeof(Elf_Sym))) return createError("SHT_SYMTAB_SHNDX section has sh_size (" + Twine(SymTable.sh_size) + @@ -662,11 +675,12 @@ ELFFile<ELFT>::getStringTableForSymtab(const Elf_Shdr &Sec, template <class ELFT> Expected<StringRef> -ELFFile<ELFT>::getSectionName(const Elf_Shdr *Section) const { +ELFFile<ELFT>::getSectionName(const Elf_Shdr *Section, + WarningHandler WarnHandler) const { auto SectionsOrErr = sections(); if (!SectionsOrErr) return SectionsOrErr.takeError(); - auto Table = getSectionStringTable(*SectionsOrErr); + auto Table = getSectionStringTable(*SectionsOrErr, WarnHandler); if (!Table) return Table.takeError(); return getSectionName(Section, *Table); diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h index 86c015efd704..424289a9ccaa 100644 --- a/include/llvm/Object/ELFObjectFile.h +++ b/include/llvm/Object/ELFObjectFile.h @@ -41,7 +41,7 @@ namespace llvm { namespace object { -constexpr int NumElfSymbolTypes = 8; +constexpr int NumElfSymbolTypes = 16; extern const llvm::EnumEntry<unsigned> ElfSymbolTypes[NumElfSymbolTypes]; class elf_symbol_iterator; @@ -239,6 +239,10 @@ public: using Elf_Rela = typename ELFT::Rela; using Elf_Dyn = typename ELFT::Dyn; + SectionRef toSectionRef(const Elf_Shdr *Sec) const { + return SectionRef(toDRI(Sec), this); + } + private: ELFObjectFile(MemoryBufferRef Object, ELFFile<ELFT> EF, const Elf_Shdr *DotDynSymSec, const Elf_Shdr *DotSymtabSec, @@ -284,7 +288,8 @@ protected: relocation_iterator section_rel_begin(DataRefImpl Sec) const override; relocation_iterator section_rel_end(DataRefImpl Sec) const override; std::vector<SectionRef> dynamic_relocation_sections() const override; - section_iterator getRelocatedSection(DataRefImpl Sec) const override; + Expected<section_iterator> + getRelocatedSection(DataRefImpl Sec) const override; void moveRelocationNext(DataRefImpl &Rel) const override; uint64_t getRelocationOffset(DataRefImpl Rel) const override; @@ -461,13 +466,15 @@ Expected<StringRef> ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Sym) const { if (!SymStrTabOrErr) return SymStrTabOrErr.takeError(); Expected<StringRef> Name = ESym->getName(*SymStrTabOrErr); + if (Name && !Name->empty()) + return Name; // If the symbol name is empty use the section name. - if ((!Name || Name->empty()) && ESym->getType() == ELF::STT_SECTION) { - StringRef SecName; - Expected<section_iterator> Sec = getSymbolSection(Sym); - if (Sec && !(*Sec)->getName(SecName)) - return SecName; + if (ESym->getType() == ELF::STT_SECTION) { + if (Expected<section_iterator> SecOrErr = getSymbolSection(Sym)) { + consumeError(Name.takeError()); + return (*SecOrErr)->getName(); + } } return Name; } @@ -835,7 +842,7 @@ ELFObjectFile<ELFT>::section_rel_end(DataRefImpl Sec) const { } template <class ELFT> -section_iterator +Expected<section_iterator> ELFObjectFile<ELFT>::getRelocatedSection(DataRefImpl Sec) const { if (EF.getHeader()->e_type != ELF::ET_REL) return section_end(); @@ -845,10 +852,10 @@ ELFObjectFile<ELFT>::getRelocatedSection(DataRefImpl Sec) const { if (Type != ELF::SHT_REL && Type != ELF::SHT_RELA) return section_end(); - auto R = EF.getSection(EShdr->sh_info); - if (!R) - report_fatal_error(errorToErrorCode(R.takeError()).message()); - return section_iterator(SectionRef(toDRI(*R), this)); + Expected<const Elf_Shdr *> SecOrErr = EF.getSection(EShdr->sh_info); + if (!SecOrErr) + return SecOrErr.takeError(); + return section_iterator(SectionRef(toDRI(*SecOrErr), this)); } // Relocations diff --git a/include/llvm/Object/ELFTypes.h b/include/llvm/Object/ELFTypes.h index 5552208b1f8a..7d1ade4d5437 100644 --- a/include/llvm/Object/ELFTypes.h +++ b/include/llvm/Object/ELFTypes.h @@ -248,7 +248,11 @@ template <class ELFT> Expected<StringRef> Elf_Sym_Impl<ELFT>::getName(StringRef StrTab) const { uint32_t Offset = this->st_name; if (Offset >= StrTab.size()) - return errorCodeToError(object_error::parse_failed); + return createStringError(object_error::parse_failed, + "st_name (0x%" PRIx32 + ") is past the end of the string table" + " of size 0x%zx", + Offset, StrTab.size()); return StringRef(StrTab.data() + Offset); } diff --git a/include/llvm/Object/MachO.h b/include/llvm/Object/MachO.h index ca9512f21706..76be8049a7d4 100644 --- a/include/llvm/Object/MachO.h +++ b/include/llvm/Object/MachO.h @@ -297,6 +297,7 @@ public: uint64_t getSectionAddress(DataRefImpl Sec) const override; uint64_t getSectionIndex(DataRefImpl Sec) const override; uint64_t getSectionSize(DataRefImpl Sec) const override; + ArrayRef<uint8_t> getSectionContents(uint32_t Offset, uint64_t Size) const; Expected<ArrayRef<uint8_t>> getSectionContents(DataRefImpl Sec) const override; uint64_t getSectionAlignment(DataRefImpl Sec) const override; diff --git a/include/llvm/Object/MachOUniversal.h b/include/llvm/Object/MachOUniversal.h index 5bf724f2c8b2..eb45aff4480b 100644 --- a/include/llvm/Object/MachOUniversal.h +++ b/include/llvm/Object/MachOUniversal.h @@ -31,6 +31,8 @@ class MachOUniversalBinary : public Binary { uint32_t Magic; uint32_t NumberOfObjects; public: + static constexpr uint32_t MaxSectionAlignment = 15; /* 2**15 or 0x8000 */ + class ObjectForArch { const MachOUniversalBinary *Parent; /// Index of object in the universal binary. @@ -64,13 +66,13 @@ public: else // Parent->getMagic() == MachO::FAT_MAGIC_64 return Header64.cpusubtype; } - uint32_t getOffset() const { + uint64_t getOffset() const { if (Parent->getMagic() == MachO::FAT_MAGIC) return Header.offset; else // Parent->getMagic() == MachO::FAT_MAGIC_64 return Header64.offset; } - uint32_t getSize() const { + uint64_t getSize() const { if (Parent->getMagic() == MachO::FAT_MAGIC) return Header.size; else // Parent->getMagic() == MachO::FAT_MAGIC_64 @@ -157,8 +159,14 @@ public: return V->isMachOUniversalBinary(); } - Expected<std::unique_ptr<MachOObjectFile>> + Expected<ObjectForArch> getObjectForArch(StringRef ArchName) const; + + Expected<std::unique_ptr<MachOObjectFile>> + getMachOObjectForArch(StringRef ArchName) const; + + Expected<std::unique_ptr<Archive>> + getArchiveForArch(StringRef ArchName) const; }; } diff --git a/include/llvm/Object/Minidump.h b/include/llvm/Object/Minidump.h index 470008d552e7..4429493aff45 100644 --- a/include/llvm/Object/Minidump.h +++ b/include/llvm/Object/Minidump.h @@ -11,6 +11,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/iterator.h" #include "llvm/BinaryFormat/Minidump.h" #include "llvm/Object/Binary.h" #include "llvm/Support/Error.h" @@ -80,16 +81,65 @@ public: return getListStream<minidump::Thread>(minidump::StreamType::ThreadList); } - /// Returns the list of memory ranges embedded in the MemoryList stream. An - /// error is returned if the file does not contain this stream, or if the - /// stream is not large enough to contain the number of memory descriptors - /// declared in the stream header. The consistency of the MemoryDescriptor - /// entries themselves is not checked in any way. + /// Returns the contents of the Exception stream. An error is returned if the + /// file does not contain this stream, or the stream is smaller than the size + /// of the ExceptionStream structure. The internal consistency of the stream + /// is not checked in any way. + Expected<const minidump::ExceptionStream &> getExceptionStream() const { + return getStream<minidump::ExceptionStream>( + minidump::StreamType::Exception); + } + + /// Returns the list of descriptors embedded in the MemoryList stream. The + /// descriptors provide the content of interesting regions of memory at the + /// time the minidump was taken. An error is returned if the file does not + /// contain this stream, or if the stream is not large enough to contain the + /// number of memory descriptors declared in the stream header. The + /// consistency of the MemoryDescriptor entries themselves is not checked in + /// any way. Expected<ArrayRef<minidump::MemoryDescriptor>> getMemoryList() const { return getListStream<minidump::MemoryDescriptor>( minidump::StreamType::MemoryList); } + class MemoryInfoIterator + : public iterator_facade_base<MemoryInfoIterator, + std::forward_iterator_tag, + minidump::MemoryInfo> { + public: + MemoryInfoIterator(ArrayRef<uint8_t> Storage, size_t Stride) + : Storage(Storage), Stride(Stride) { + assert(Storage.size() % Stride == 0); + } + + bool operator==(const MemoryInfoIterator &R) const { + return Storage.size() == R.Storage.size(); + } + + const minidump::MemoryInfo &operator*() const { + assert(Storage.size() >= sizeof(minidump::MemoryInfo)); + return *reinterpret_cast<const minidump::MemoryInfo *>(Storage.data()); + } + + MemoryInfoIterator &operator++() { + Storage = Storage.drop_front(Stride); + return *this; + } + + private: + ArrayRef<uint8_t> Storage; + size_t Stride; + }; + + /// Returns the list of descriptors embedded in the MemoryInfoList stream. The + /// descriptors provide properties (e.g. permissions) of interesting regions + /// of memory at the time the minidump was taken. An error is returned if the + /// file does not contain this stream, or if the stream is not large enough to + /// contain the number of memory descriptors declared in the stream header. + /// The consistency of the MemoryInfoList entries themselves is not checked + /// in any way. + Expected<iterator_range<MemoryInfoIterator>> getMemoryInfoList() const; + private: static Error createError(StringRef Str) { return make_error<GenericBinaryError>(Str, object_error::parse_failed); @@ -137,10 +187,10 @@ private: }; template <typename T> -Expected<const T &> MinidumpFile::getStream(minidump::StreamType Stream) const { - if (auto OptionalStream = getRawStream(Stream)) { - if (OptionalStream->size() >= sizeof(T)) - return *reinterpret_cast<const T *>(OptionalStream->data()); +Expected<const T &> MinidumpFile::getStream(minidump::StreamType Type) const { + if (Optional<ArrayRef<uint8_t>> Stream = getRawStream(Type)) { + if (Stream->size() >= sizeof(T)) + return *reinterpret_cast<const T *>(Stream->data()); return createEOFError(); } return createError("No such stream"); @@ -153,10 +203,11 @@ Expected<ArrayRef<T>> MinidumpFile::getDataSliceAs(ArrayRef<uint8_t> Data, // Check for overflow. if (Count > std::numeric_limits<size_t>::max() / sizeof(T)) return createEOFError(); - auto ExpectedArray = getDataSlice(Data, Offset, sizeof(T) * Count); - if (!ExpectedArray) - return ExpectedArray.takeError(); - return ArrayRef<T>(reinterpret_cast<const T *>(ExpectedArray->data()), Count); + Expected<ArrayRef<uint8_t>> Slice = + getDataSlice(Data, Offset, sizeof(T) * Count); + if (!Slice) + return Slice.takeError(); + return ArrayRef<T>(reinterpret_cast<const T *>(Slice->data()), Count); } } // end namespace object diff --git a/include/llvm/Object/ObjectFile.h b/include/llvm/Object/ObjectFile.h index 483a3486bd72..adc9dbc189af 100644 --- a/include/llvm/Object/ObjectFile.h +++ b/include/llvm/Object/ObjectFile.h @@ -94,7 +94,7 @@ public: void moveNext(); - std::error_code getName(StringRef &Result) const; + Expected<StringRef> getName() const; uint64_t getAddress() const; uint64_t getIndex() const; uint64_t getSize() const; @@ -130,18 +130,13 @@ public: iterator_range<relocation_iterator> relocations() const { return make_range(relocation_begin(), relocation_end()); } - section_iterator getRelocatedSection() const; + Expected<section_iterator> getRelocatedSection() const; DataRefImpl getRawDataRefImpl() const; const ObjectFile *getObject() const; }; struct SectionedAddress { - // TODO: constructors could be removed when C++14 would be adopted. - SectionedAddress() {} - SectionedAddress(uint64_t Addr, uint64_t SectIdx) - : Address(Addr), SectionIndex(SectIdx) {} - const static uint64_t UndefSection = UINT64_MAX; uint64_t Address = 0; @@ -277,7 +272,7 @@ protected: virtual bool isBerkeleyData(DataRefImpl Sec) const; virtual relocation_iterator section_rel_begin(DataRefImpl Sec) const = 0; virtual relocation_iterator section_rel_end(DataRefImpl Sec) const = 0; - virtual section_iterator getRelocatedSection(DataRefImpl Sec) const; + virtual Expected<section_iterator> getRelocatedSection(DataRefImpl Sec) const; // Same as above for RelocationRef. friend class RelocationRef; @@ -434,12 +429,8 @@ inline void SectionRef::moveNext() { return OwningObject->moveSectionNext(SectionPimpl); } -inline std::error_code SectionRef::getName(StringRef &Result) const { - Expected<StringRef> NameOrErr = OwningObject->getSectionName(SectionPimpl); - if (!NameOrErr) - return errorToErrorCode(NameOrErr.takeError()); - Result = *NameOrErr; - return std::error_code(); +inline Expected<StringRef> SectionRef::getName() const { + return OwningObject->getSectionName(SectionPimpl); } inline uint64_t SectionRef::getAddress() const { @@ -510,7 +501,7 @@ inline relocation_iterator SectionRef::relocation_end() const { return OwningObject->section_rel_end(SectionPimpl); } -inline section_iterator SectionRef::getRelocatedSection() const { +inline Expected<section_iterator> SectionRef::getRelocatedSection() const { return OwningObject->getRelocatedSection(SectionPimpl); } diff --git a/include/llvm/Object/StackMapParser.h b/include/llvm/Object/StackMapParser.h index ed44efbf80b9..b408f4041034 100644 --- a/include/llvm/Object/StackMapParser.h +++ b/include/llvm/Object/StackMapParser.h @@ -19,7 +19,7 @@ namespace llvm { -/// A parser for the latest stackmap format. At the moment, latest=V2. +/// A parser for the latest stackmap format. At the moment, latest=V3. template <support::endianness Endianness> class StackMapParser { public: @@ -299,7 +299,7 @@ public: const uint8_t *P; }; - /// Construct a parser for a version-2 stackmap. StackMap data will be read + /// Construct a parser for a version-3 stackmap. StackMap data will be read /// from the given array. StackMapParser(ArrayRef<uint8_t> StackMapSection) : StackMapSection(StackMapSection) { diff --git a/include/llvm/Object/TapiFile.h b/include/llvm/Object/TapiFile.h new file mode 100644 index 000000000000..bc2e04e1cc96 --- /dev/null +++ b/include/llvm/Object/TapiFile.h @@ -0,0 +1,60 @@ +//===- TapiFile.h - Text-based Dynamic Library Stub -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the TapiFile interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_TAPI_FILE_H +#define LLVM_OBJECT_TAPI_FILE_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TextAPI/MachO/InterfaceFile.h" + +namespace llvm { +namespace object { + +class TapiFile : public SymbolicFile { +public: + TapiFile(MemoryBufferRef Source, const MachO::InterfaceFile &interface, + MachO::Architecture Arch); + ~TapiFile() override; + + void moveSymbolNext(DataRefImpl &DRI) const override; + + Error printSymbolName(raw_ostream &OS, DataRefImpl DRI) const override; + + uint32_t getSymbolFlags(DataRefImpl DRI) const override; + + basic_symbol_iterator symbol_begin() const override; + + basic_symbol_iterator symbol_end() const override; + + static bool classof(const Binary *v) { return v->isTapiFile(); } + +private: + struct Symbol { + StringRef Prefix; + StringRef Name; + uint32_t Flags; + + constexpr Symbol(StringRef Prefix, StringRef Name, uint32_t Flags) + : Prefix(Prefix), Name(Name), Flags(Flags) {} + }; + + std::vector<Symbol> Symbols; +}; + +} // end namespace object. +} // end namespace llvm. + +#endif // LLVM_OBJECT_TAPI_FILE_H diff --git a/include/llvm/Object/TapiUniversal.h b/include/llvm/Object/TapiUniversal.h new file mode 100644 index 000000000000..4931183852ad --- /dev/null +++ b/include/llvm/Object/TapiUniversal.h @@ -0,0 +1,109 @@ +//===-- TapiUniversal.h - Text-based Dynamic Library Stub -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the TapiUniversal interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_TAPI_UNIVERSAL_H +#define LLVM_OBJECT_TAPI_UNIVERSAL_H + +#include "llvm/Object/Binary.h" +#include "llvm/Object/TapiFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TextAPI/MachO/Architecture.h" +#include "llvm/TextAPI/MachO/InterfaceFile.h" + +namespace llvm { +namespace object { + +class TapiUniversal : public Binary { +public: + class ObjectForArch { + const TapiUniversal *Parent; + int Index; + + public: + ObjectForArch(const TapiUniversal *Parent, int Index) + : Parent(Parent), Index(Index) {} + + ObjectForArch getNext() const { return ObjectForArch(Parent, Index + 1); } + + bool operator==(const ObjectForArch &Other) const { + return (Parent == Other.Parent) && (Index == Other.Index); + } + + uint32_t getCPUType() const { + auto Result = + MachO::getCPUTypeFromArchitecture(Parent->Architectures[Index]); + return Result.first; + } + + uint32_t getCPUSubType() const { + auto Result = + MachO::getCPUTypeFromArchitecture(Parent->Architectures[Index]); + return Result.second; + } + + std::string getArchFlagName() const { + return MachO::getArchitectureName(Parent->Architectures[Index]); + } + + Expected<std::unique_ptr<TapiFile>> getAsObjectFile() const; + }; + + class object_iterator { + ObjectForArch Obj; + + public: + object_iterator(const ObjectForArch &Obj) : Obj(Obj) {} + const ObjectForArch *operator->() const { return &Obj; } + const ObjectForArch &operator*() const { return Obj; } + + bool operator==(const object_iterator &Other) const { + return Obj == Other.Obj; + } + bool operator!=(const object_iterator &Other) const { + return !(*this == Other); + } + + object_iterator &operator++() { // Preincrement + Obj = Obj.getNext(); + return *this; + } + }; + + TapiUniversal(MemoryBufferRef Source, Error &Err); + static Expected<std::unique_ptr<TapiUniversal>> + create(MemoryBufferRef Source); + ~TapiUniversal() override; + + object_iterator begin_objects() const { return ObjectForArch(this, 0); } + object_iterator end_objects() const { + return ObjectForArch(this, Architectures.size()); + } + + iterator_range<object_iterator> objects() const { + return make_range(begin_objects(), end_objects()); + } + + uint32_t getNumberOfObjects() const { return Architectures.size(); } + + // Cast methods. + static bool classof(const Binary *v) { return v->isTapiUniversal(); } + +private: + std::unique_ptr<MachO::InterfaceFile> ParsedFile; + std::vector<MachO::Architecture> Architectures; +}; + +} // end namespace object. +} // end namespace llvm. + +#endif // LLVM_OBJECT_TAPI_UNIVERSAL_H diff --git a/include/llvm/Object/WindowsResource.h b/include/llvm/Object/WindowsResource.h index 356dcb03abba..a0d658491cb9 100644 --- a/include/llvm/Object/WindowsResource.h +++ b/include/llvm/Object/WindowsResource.h @@ -31,6 +31,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" #include "llvm/Object/Error.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryStreamReader.h" @@ -48,6 +49,7 @@ class ScopedPrinter; namespace object { class WindowsResource; +class ResourceSectionRef; const size_t WIN_RES_MAGIC_SIZE = 16; const size_t WIN_RES_NULL_ENTRY_SIZE = 16; @@ -151,8 +153,11 @@ private: class WindowsResourceParser { public: class TreeNode; - WindowsResourceParser(); + WindowsResourceParser(bool MinGW = false); Error parse(WindowsResource *WR, std::vector<std::string> &Duplicates); + Error parse(ResourceSectionRef &RSR, StringRef Filename, + std::vector<std::string> &Duplicates); + void cleanUpManifests(std::vector<std::string> &Duplicates); void printTree(raw_ostream &OS) const; const TreeNode &getTree() const { return Root; } const ArrayRef<std::vector<uint8_t>> getData() const { return Data; } @@ -181,32 +186,38 @@ public: private: friend class WindowsResourceParser; - static uint32_t StringCount; - static uint32_t DataCount; - - static std::unique_ptr<TreeNode> createStringNode(); + // Index is the StringTable vector index for this node's name. + static std::unique_ptr<TreeNode> createStringNode(uint32_t Index); static std::unique_ptr<TreeNode> createIDNode(); + // DataIndex is the Data vector index that the data node points at. static std::unique_ptr<TreeNode> createDataNode(uint16_t MajorVersion, uint16_t MinorVersion, uint32_t Characteristics, - uint32_t Origin); + uint32_t Origin, + uint32_t DataIndex); - explicit TreeNode(bool IsStringNode); + explicit TreeNode(uint32_t StringIndex); TreeNode(uint16_t MajorVersion, uint16_t MinorVersion, - uint32_t Characteristics, uint32_t Origin); + uint32_t Characteristics, uint32_t Origin, uint32_t DataIndex); bool addEntry(const ResourceEntryRef &Entry, uint32_t Origin, - bool &IsNewTypeString, bool &IsNewNameString, + std::vector<std::vector<uint8_t>> &Data, + std::vector<std::vector<UTF16>> &StringTable, TreeNode *&Result); - TreeNode &addTypeNode(const ResourceEntryRef &Entry, bool &IsNewTypeString); - TreeNode &addNameNode(const ResourceEntryRef &Entry, bool &IsNewNameString); + TreeNode &addTypeNode(const ResourceEntryRef &Entry, + std::vector<std::vector<UTF16>> &StringTable); + TreeNode &addNameNode(const ResourceEntryRef &Entry, + std::vector<std::vector<UTF16>> &StringTable); bool addLanguageNode(const ResourceEntryRef &Entry, uint32_t Origin, + std::vector<std::vector<uint8_t>> &Data, TreeNode *&Result); bool addDataChild(uint32_t ID, uint16_t MajorVersion, uint16_t MinorVersion, uint32_t Characteristics, uint32_t Origin, - TreeNode *&Result); + uint32_t DataIndex, TreeNode *&Result); TreeNode &addIDChild(uint32_t ID); - TreeNode &addNameChild(ArrayRef<UTF16> NameRef, bool &IsNewString); + TreeNode &addNameChild(ArrayRef<UTF16> NameRef, + std::vector<std::vector<UTF16>> &StringTable); + void shiftDataIndexDown(uint32_t Index); bool IsDataNode = false; uint32_t StringIndex; @@ -222,12 +233,30 @@ public: uint32_t Origin; }; + struct StringOrID { + bool IsString; + ArrayRef<UTF16> String; + uint32_t ID; + + StringOrID(uint32_t ID) : IsString(false), ID(ID) {} + StringOrID(ArrayRef<UTF16> String) : IsString(true), String(String) {} + }; + private: + Error addChildren(TreeNode &Node, ResourceSectionRef &RSR, + const coff_resource_dir_table &Table, uint32_t Origin, + std::vector<StringOrID> &Context, + std::vector<std::string> &Duplicates); + bool shouldIgnoreDuplicate(const ResourceEntryRef &Entry) const; + bool shouldIgnoreDuplicate(const std::vector<StringOrID> &Context) const; + TreeNode Root; std::vector<std::vector<uint8_t>> Data; std::vector<std::vector<UTF16>> StringTable; std::vector<std::string> InputFilenames; + + bool MinGW; }; Expected<std::unique_ptr<MemoryBuffer>> diff --git a/include/llvm/Object/XCOFFObjectFile.h b/include/llvm/Object/XCOFFObjectFile.h index cdee7129a2ab..84073ce5f6cf 100644 --- a/include/llvm/Object/XCOFFObjectFile.h +++ b/include/llvm/Object/XCOFFObjectFile.h @@ -13,23 +13,8 @@ #ifndef LLVM_OBJECT_XCOFFOBJECTFILE_H #define LLVM_OBJECT_XCOFFOBJECTFILE_H -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/iterator_range.h" -#include "llvm/BinaryFormat/Magic.h" #include "llvm/BinaryFormat/XCOFF.h" -#include "llvm/MC/SubtargetFeature.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/Error.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Object/SymbolicFile.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Error.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/MemoryBuffer.h" -#include <cassert> -#include <cstdint> -#include <memory> -#include <system_error> namespace llvm { namespace object { @@ -63,7 +48,7 @@ struct XCOFFFileHeader64 { }; struct XCOFFSectionHeader32 { - char Name[XCOFF::SectionNameSize]; + char Name[XCOFF::NameSize]; support::ubig32_t PhysicalAddress; support::ubig32_t VirtualAddress; support::ubig32_t SectionSize; @@ -78,7 +63,7 @@ struct XCOFFSectionHeader32 { }; struct XCOFFSectionHeader64 { - char Name[XCOFF::SectionNameSize]; + char Name[XCOFF::NameSize]; support::ubig64_t PhysicalAddress; support::ubig64_t VirtualAddress; support::ubig64_t SectionSize; @@ -106,7 +91,7 @@ struct XCOFFSymbolEntry { } CFileLanguageIdAndTypeIdType; union { - char SymbolName[XCOFF::SymbolNameSize]; + char SymbolName[XCOFF::NameSize]; NameInStrTblType NameInStrTbl; }; @@ -127,6 +112,75 @@ struct XCOFFStringTable { const char *Data; }; +struct XCOFFCsectAuxEnt32 { + support::ubig32_t + SectionOrLength; // If the symbol type is XTY_SD or XTY_CM, the csect + // length. + // If the symbol type is XTY_LD, the symbol table + // index of the containing csect. + // If the symbol type is XTY_ER, 0. + support::ubig32_t ParameterHashIndex; + support::ubig16_t TypeChkSectNum; + uint8_t SymbolAlignmentAndType; + XCOFF::StorageMappingClass StorageMappingClass; + support::ubig32_t StabInfoIndex; + support::ubig16_t StabSectNum; +}; + +struct XCOFFFileAuxEnt { + typedef struct { + support::big32_t Magic; // Zero indicates name in string table. + support::ubig32_t Offset; + char NamePad[XCOFF::FileNamePadSize]; + } NameInStrTblType; + union { + char Name[XCOFF::NameSize + XCOFF::FileNamePadSize]; + NameInStrTblType NameInStrTbl; + }; + XCOFF::CFileStringType Type; + uint8_t ReservedZeros[2]; + uint8_t AuxType; // 64-bit XCOFF file only. +}; + +struct XCOFFSectAuxEntForStat { + support::ubig32_t SectionLength; + support::ubig16_t NumberOfRelocEnt; + support::ubig16_t NumberOfLineNum; + uint8_t Pad[10]; +}; + +struct XCOFFRelocation32 { + // Masks for packing/unpacking the r_rsize field of relocations. + + // The msb is used to indicate if the bits being relocated are signed or + // unsigned. + static constexpr uint8_t XR_SIGN_INDICATOR_MASK = 0x80; + + // The 2nd msb is used to indicate that the binder has replaced/modified the + // original instruction. + static constexpr uint8_t XR_FIXUP_INDICATOR_MASK = 0x40; + + // The remaining bits specify the bit length of the relocatable reference + // minus one. + static constexpr uint8_t XR_BIASED_LENGTH_MASK = 0x3f; + +public: + support::ubig32_t VirtualAddress; + support::ubig32_t SymbolIndex; + + // Packed field, see XR_* masks for details of packing. + uint8_t Info; + + XCOFF::RelocationType Type; + +public: + bool isRelocationSigned() const; + bool isFixupIndicated() const; + + // Returns the number of bits being relocated. + uint8_t getRelocatedLength() const; +}; + class XCOFFObjectFile : public ObjectFile { private: const void *FileHeader = nullptr; @@ -146,18 +200,18 @@ private: const XCOFFSectionHeader32 *toSection32(DataRefImpl Ref) const; const XCOFFSectionHeader64 *toSection64(DataRefImpl Ref) const; - void checkSectionAddress(uintptr_t Addr, uintptr_t TableAddr) const; uintptr_t getSectionHeaderTableAddress() const; + uintptr_t getEndOfSymbolTableAddress() const; // This returns a pointer to the start of the storage for the name field of // the 32-bit or 64-bit SectionHeader struct. This string is *not* necessarily // null-terminated. const char *getSectionNameInternal(DataRefImpl Sec) const; - int32_t getSectionFlags(DataRefImpl Sec) const; + // This function returns string table entry. + Expected<StringRef> getStringTableEntry(uint32_t Offset) const; static bool isReservedSectionNumber(int16_t SectionNumber); - Expected<DataRefImpl> getSectionByNum(int16_t Num) const; // Constructor and "create" factory function. The constructor is only a thin // wrapper around the base constructor. The "create" function fills out the @@ -175,6 +229,8 @@ private: friend Expected<std::unique_ptr<ObjectFile>> ObjectFile::createXCOFFObjectFile(MemoryBufferRef Object, unsigned FileType); + void checkSectionAddress(uintptr_t Addr, uintptr_t TableAddr) const; + public: // Interface inherited from base classes. void moveSymbolNext(DataRefImpl &Symb) const override; @@ -253,15 +309,49 @@ public: uint32_t getLogicalNumberOfSymbolTableEntries32() const; uint32_t getNumberOfSymbolTableEntries64() const; + uint32_t getSymbolIndex(uintptr_t SymEntPtr) const; + Expected<StringRef> getSymbolNameByIndex(uint32_t SymbolTableIndex) const; + Expected<StringRef> getCFileName(const XCOFFFileAuxEnt *CFileEntPtr) const; uint16_t getOptionalHeaderSize() const; uint16_t getFlags() const; // Section header table related interfaces. ArrayRef<XCOFFSectionHeader32> sections32() const; ArrayRef<XCOFFSectionHeader64> sections64() const; + + int32_t getSectionFlags(DataRefImpl Sec) const; + Expected<DataRefImpl> getSectionByNum(int16_t Num) const; + + void checkSymbolEntryPointer(uintptr_t SymbolEntPtr) const; + + // Relocation-related interfaces. + Expected<uint32_t> + getLogicalNumberOfRelocationEntries(const XCOFFSectionHeader32 &Sec) const; + + Expected<ArrayRef<XCOFFRelocation32>> + relocations(const XCOFFSectionHeader32 &) const; }; // XCOFFObjectFile +class XCOFFSymbolRef { + const DataRefImpl SymEntDataRef; + const XCOFFObjectFile *const OwningObjectPtr; + +public: + XCOFFSymbolRef(DataRefImpl SymEntDataRef, + const XCOFFObjectFile *OwningObjectPtr) + : SymEntDataRef(SymEntDataRef), OwningObjectPtr(OwningObjectPtr){}; + + XCOFF::StorageClass getStorageClass() const; + uint8_t getNumberOfAuxEntries() const; + const XCOFFCsectAuxEnt32 *getXCOFFCsectAuxEnt32() const; + uint16_t getType() const; + int16_t getSectionNumber() const; + + bool hasCsectAuxEnt() const; + bool isFunction() const; +}; + } // namespace object } // namespace llvm diff --git a/include/llvm/ObjectYAML/DWARFYAML.h b/include/llvm/ObjectYAML/DWARFYAML.h index 78d736c3ef05..525fd9a89242 100644 --- a/include/llvm/ObjectYAML/DWARFYAML.h +++ b/include/llvm/ObjectYAML/DWARFYAML.h @@ -234,7 +234,7 @@ template <> struct MappingTraits<DWARFYAML::InitialLength> { static void mapping(IO &IO, DWARFYAML::InitialLength &DWARF); }; -#define HANDLE_DW_TAG(unused, name, unused2, unused3) \ +#define HANDLE_DW_TAG(unused, name, unused2, unused3, unused4) \ io.enumCase(value, "DW_TAG_" #name, dwarf::DW_TAG_##name); template <> struct ScalarEnumerationTraits<dwarf::Tag> { diff --git a/include/llvm/ObjectYAML/ELFYAML.h b/include/llvm/ObjectYAML/ELFYAML.h index f4212516f486..0898a0e7d532 100644 --- a/include/llvm/ObjectYAML/ELFYAML.h +++ b/include/llvm/ObjectYAML/ELFYAML.h @@ -25,6 +25,8 @@ namespace llvm { namespace ELFYAML { +StringRef dropUniqueSuffix(StringRef S); + // These types are invariant across 32/64-bit ELF, so for simplicity just // directly give them their exact sizes. We don't need to worry about // endianness because these are just the types in the YAMLIO structures, @@ -54,8 +56,6 @@ LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_SHF) LLVM_YAML_STRONG_TYPEDEF(uint16_t, ELF_SHN) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STB) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STT) -LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STV) -LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STO) LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_AFL_REG) LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_ABI_FP) @@ -77,7 +77,7 @@ struct FileHeader { llvm::yaml::Hex64 Entry; Optional<llvm::yaml::Hex16> SHEntSize; - Optional<llvm::yaml::Hex16> SHOffset; + Optional<llvm::yaml::Hex64> SHOff; Optional<llvm::yaml::Hex16> SHNum; Optional<llvm::yaml::Hex16> SHStrNdx; }; @@ -107,7 +107,7 @@ struct Symbol { ELF_STB Binding; llvm::yaml::Hex64 Value; llvm::yaml::Hex64 Size; - uint8_t Other; + Optional<uint8_t> Other; }; struct SectionOrType { @@ -119,6 +119,11 @@ struct DynamicEntry { llvm::yaml::Hex64 Val; }; +struct StackSizeEntry { + llvm::yaml::Hex64 Address; + llvm::yaml::Hex64 Size; +}; + struct Section { enum class SectionKind { Dynamic, @@ -126,10 +131,14 @@ struct Section { RawContent, Relocation, NoBits, + Hash, Verdef, Verneed, + StackSizes, + SymtabShndxSection, Symver, - MipsABIFlags + MipsABIFlags, + Addrsig }; SectionKind Kind; StringRef Name; @@ -140,16 +149,44 @@ struct Section { llvm::yaml::Hex64 AddressAlign; Optional<llvm::yaml::Hex64> EntSize; + // Usually sections are not created implicitly, but loaded from YAML. + // When they are, this flag is used to signal about that. + bool IsImplicit; + + Section(SectionKind Kind, bool IsImplicit = false) + : Kind(Kind), IsImplicit(IsImplicit) {} + virtual ~Section(); + + // The following members are used to override section fields which is + // useful for creating invalid objects. + + // This can be used to override the offset stored in the sh_name field. + // It does not affect the name stored in the string table. + Optional<llvm::yaml::Hex64> ShName; + // This can be used to override the sh_offset field. It does not place the - // section data at the offset specified. Useful for creating invalid objects. + // section data at the offset specified. Optional<llvm::yaml::Hex64> ShOffset; // This can be used to override the sh_size field. It does not affect the // content written. Optional<llvm::yaml::Hex64> ShSize; +}; - Section(SectionKind Kind) : Kind(Kind) {} - virtual ~Section(); +struct StackSizesSection : Section { + Optional<yaml::BinaryRef> Content; + Optional<llvm::yaml::Hex64> Size; + Optional<std::vector<StackSizeEntry>> Entries; + + StackSizesSection() : Section(SectionKind::StackSizes) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::StackSizes; + } + + static bool nameMatches(StringRef Name) { + return Name == ".stack_sizes"; + } }; struct DynamicSection : Section { @@ -185,6 +222,17 @@ struct NoBitsSection : Section { } }; +struct HashSection : Section { + Optional<yaml::BinaryRef> Content; + Optional<llvm::yaml::Hex64> Size; + Optional<std::vector<uint32_t>> Bucket; + Optional<std::vector<uint32_t>> Chain; + + HashSection() : Section(SectionKind::Hash) {} + + static bool classof(const Section *S) { return S->Kind == SectionKind::Hash; } +}; + struct VernauxEntry { uint32_t Hash; uint16_t Flags; @@ -209,6 +257,26 @@ struct VerneedSection : Section { } }; +struct AddrsigSymbol { + AddrsigSymbol(StringRef N) : Name(N), Index(None) {} + AddrsigSymbol(llvm::yaml::Hex32 Ndx) : Name(None), Index(Ndx) {} + AddrsigSymbol() : Name(None), Index(None) {} + + Optional<StringRef> Name; + Optional<llvm::yaml::Hex32> Index; +}; + +struct AddrsigSection : Section { + Optional<yaml::BinaryRef> Content; + Optional<llvm::yaml::Hex64> Size; + Optional<std::vector<AddrsigSymbol>> Symbols; + + AddrsigSection() : Section(SectionKind::Addrsig) {} + static bool classof(const Section *S) { + return S->Kind == SectionKind::Addrsig; + } +}; + struct SymverSection : Section { std::vector<uint16_t> Entries; @@ -269,6 +337,16 @@ struct RelocationSection : Section { } }; +struct SymtabShndxSection : Section { + std::vector<uint32_t> Entries; + + SymtabShndxSection() : Section(SectionKind::SymtabShndxSection) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::SymtabShndxSection; + } +}; + // Represents .MIPS.abiflags section struct MipsABIFlags : Section { llvm::yaml::Hex16 Version; @@ -298,13 +376,15 @@ struct Object { // cleaner and nicer if we read them from the YAML as a separate // top-level key, which automatically ensures that invariants like there // being a single SHT_SYMTAB section are upheld. - std::vector<Symbol> Symbols; + Optional<std::vector<Symbol>> Symbols; std::vector<Symbol> DynamicSymbols; }; } // end namespace ELFYAML } // end namespace llvm +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::AddrsigSymbol) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::StackSizeEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::DynamicEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader) LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Section>) @@ -381,16 +461,6 @@ struct ScalarEnumerationTraits<ELFYAML::ELF_STT> { }; template <> -struct ScalarEnumerationTraits<ELFYAML::ELF_STV> { - static void enumeration(IO &IO, ELFYAML::ELF_STV &Value); -}; - -template <> -struct ScalarBitSetTraits<ELFYAML::ELF_STO> { - static void bitset(IO &IO, ELFYAML::ELF_STO &Value); -}; - -template <> struct ScalarEnumerationTraits<ELFYAML::ELF_REL> { static void enumeration(IO &IO, ELFYAML::ELF_REL &Value); }; @@ -450,6 +520,10 @@ struct MappingTraits<ELFYAML::Symbol> { static StringRef validate(IO &IO, ELFYAML::Symbol &Symbol); }; +template <> struct MappingTraits<ELFYAML::StackSizeEntry> { + static void mapping(IO &IO, ELFYAML::StackSizeEntry &Rel); +}; + template <> struct MappingTraits<ELFYAML::DynamicEntry> { static void mapping(IO &IO, ELFYAML::DynamicEntry &Rel); }; @@ -466,6 +540,10 @@ template <> struct MappingTraits<ELFYAML::VernauxEntry> { static void mapping(IO &IO, ELFYAML::VernauxEntry &E); }; +template <> struct MappingTraits<ELFYAML::AddrsigSymbol> { + static void mapping(IO &IO, ELFYAML::AddrsigSymbol &Sym); +}; + template <> struct MappingTraits<ELFYAML::Relocation> { static void mapping(IO &IO, ELFYAML::Relocation &Rel); }; diff --git a/include/llvm/ObjectYAML/MachOYAML.h b/include/llvm/ObjectYAML/MachOYAML.h index d7e1c033f43b..327c3b9f892b 100644 --- a/include/llvm/ObjectYAML/MachOYAML.h +++ b/include/llvm/ObjectYAML/MachOYAML.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/ObjectYAML/DWARFYAML.h" +#include "llvm/ObjectYAML/YAML.h" #include "llvm/Support/YAMLTraits.h" #include <cstdint> #include <string> @@ -39,6 +40,7 @@ struct Section { llvm::yaml::Hex32 reserved1; llvm::yaml::Hex32 reserved2; llvm::yaml::Hex32 reserved3; + Optional<llvm::yaml::BinaryRef> content; }; struct FileHeader { @@ -198,6 +200,7 @@ template <> struct MappingTraits<MachOYAML::ExportEntry> { template <> struct MappingTraits<MachOYAML::Section> { static void mapping(IO &IO, MachOYAML::Section &Section); + static StringRef validate(IO &io, MachOYAML::Section &Section); }; template <> struct MappingTraits<MachOYAML::NListEntry> { diff --git a/include/llvm/ObjectYAML/MinidumpYAML.h b/include/llvm/ObjectYAML/MinidumpYAML.h index 39fdd62e017b..c1711a28dd84 100644 --- a/include/llvm/ObjectYAML/MinidumpYAML.h +++ b/include/llvm/ObjectYAML/MinidumpYAML.h @@ -26,6 +26,8 @@ namespace MinidumpYAML { /// from Types to Kinds is fixed and given by the static getKind function. struct Stream { enum class StreamKind { + Exception, + MemoryInfoList, MemoryList, ModuleList, RawContent, @@ -102,6 +104,45 @@ using ModuleListStream = detail::ListStream<detail::ParsedModule>; using ThreadListStream = detail::ListStream<detail::ParsedThread>; using MemoryListStream = detail::ListStream<detail::ParsedMemoryDescriptor>; +/// ExceptionStream minidump stream. +struct ExceptionStream : public Stream { + minidump::ExceptionStream MDExceptionStream; + yaml::BinaryRef ThreadContext; + + ExceptionStream() + : Stream(StreamKind::Exception, minidump::StreamType::Exception), + MDExceptionStream({}) {} + + explicit ExceptionStream(const minidump::ExceptionStream &MDExceptionStream, + ArrayRef<uint8_t> ThreadContext) + : Stream(StreamKind::Exception, minidump::StreamType::Exception), + MDExceptionStream(MDExceptionStream), ThreadContext(ThreadContext) {} + + static bool classof(const Stream *S) { + return S->Kind == StreamKind::Exception; + } +}; + +/// A structure containing the list of MemoryInfo entries comprising a +/// MemoryInfoList stream. +struct MemoryInfoListStream : public Stream { + std::vector<minidump::MemoryInfo> Infos; + + MemoryInfoListStream() + : Stream(StreamKind::MemoryInfoList, + minidump::StreamType::MemoryInfoList) {} + + explicit MemoryInfoListStream( + iterator_range<object::MinidumpFile::MemoryInfoIterator> Range) + : Stream(StreamKind::MemoryInfoList, + minidump::StreamType::MemoryInfoList), + Infos(Range.begin(), Range.end()) {} + + static bool classof(const Stream *S) { + return S->Kind == StreamKind::MemoryInfoList; + } +}; + /// A minidump stream represented as a sequence of hex bytes. This is used as a /// fallback when no other stream kind is suitable. struct RawContentStream : public Stream { @@ -122,16 +163,16 @@ struct SystemInfoStream : public Stream { minidump::SystemInfo Info; std::string CSDVersion; - explicit SystemInfoStream(const minidump::SystemInfo &Info, - std::string CSDVersion) - : Stream(StreamKind::SystemInfo, minidump::StreamType::SystemInfo), - Info(Info), CSDVersion(std::move(CSDVersion)) {} - SystemInfoStream() : Stream(StreamKind::SystemInfo, minidump::StreamType::SystemInfo) { memset(&Info, 0, sizeof(Info)); } + explicit SystemInfoStream(const minidump::SystemInfo &Info, + std::string CSDVersion) + : Stream(StreamKind::SystemInfo, minidump::StreamType::SystemInfo), + Info(Info), CSDVersion(std::move(CSDVersion)) {} + static bool classof(const Stream *S) { return S->Kind == StreamKind::SystemInfo; } @@ -177,12 +218,6 @@ struct Object { static Expected<Object> create(const object::MinidumpFile &File); }; -/// Serialize the minidump file represented by Obj to OS in binary form. -void writeAsBinary(Object &Obj, raw_ostream &OS); - -/// Serialize the yaml string as a minidump file to OS in binary form. -Error writeAsBinary(StringRef Yaml, raw_ostream &OS); - } // namespace MinidumpYAML namespace yaml { @@ -213,6 +248,10 @@ template <> struct MappingContextTraits<minidump::MemoryDescriptor, BinaryRef> { } // namespace llvm +LLVM_YAML_DECLARE_BITSET_TRAITS(llvm::minidump::MemoryProtection) +LLVM_YAML_DECLARE_BITSET_TRAITS(llvm::minidump::MemoryState) +LLVM_YAML_DECLARE_BITSET_TRAITS(llvm::minidump::MemoryType) + LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::ProcessorArchitecture) LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::OSPlatform) LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::StreamType) @@ -220,6 +259,8 @@ LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::StreamType) LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::ArmInfo) LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::OtherInfo) LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::X86Info) +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::Exception) +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::MemoryInfo) LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::VSFixedFileInfo) LLVM_YAML_DECLARE_MAPPING_TRAITS( @@ -233,6 +274,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::MinidumpYAML::Stream>) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::MemoryListStream::entry_type) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ModuleListStream::entry_type) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ThreadListStream::entry_type) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::minidump::MemoryInfo) LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::MinidumpYAML::Object) diff --git a/include/llvm/ObjectYAML/WasmYAML.h b/include/llvm/ObjectYAML/WasmYAML.h index 2411dc7ac17d..15a8cc215020 100644 --- a/include/llvm/ObjectYAML/WasmYAML.h +++ b/include/llvm/ObjectYAML/WasmYAML.h @@ -145,7 +145,7 @@ struct Signature { uint32_t Index; SignatureForm Form = wasm::WASM_TYPE_FUNC; std::vector<ValueType> ParamTypes; - ValueType ReturnType; + std::vector<ValueType> ReturnTypes; }; struct SymbolInfo { diff --git a/include/llvm/ObjectYAML/yaml2obj.h b/include/llvm/ObjectYAML/yaml2obj.h new file mode 100644 index 000000000000..386551337d86 --- /dev/null +++ b/include/llvm/ObjectYAML/yaml2obj.h @@ -0,0 +1,67 @@ +//===--- yaml2obj.h - -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Common declarations for yaml2obj +//===----------------------------------------------------------------------===// +#ifndef LLVM_TOOLS_YAML2OBJ_YAML2OBJ_H +#define LLVM_TOOLS_YAML2OBJ_YAML2OBJ_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include <memory> + +namespace llvm { +class raw_ostream; +template <typename T> class SmallVectorImpl; +template <typename T> class Expected; + +namespace object { +class ObjectFile; +} + +namespace COFFYAML { +struct Object; +} + +namespace ELFYAML { +struct Object; +} + +namespace MinidumpYAML { +struct Object; +} + +namespace WasmYAML { +struct Object; +} + +namespace yaml { +class Input; +struct YamlObjectFile; + +using ErrorHandler = llvm::function_ref<void(const Twine &Msg)>; + +bool yaml2coff(COFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); +bool yaml2elf(ELFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); +bool yaml2macho(YamlObjectFile &Doc, raw_ostream &Out, ErrorHandler EH); +bool yaml2minidump(MinidumpYAML::Object &Doc, raw_ostream &Out, + ErrorHandler EH); +bool yaml2wasm(WasmYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); + +bool convertYAML(Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler, + unsigned DocNum = 1); + +/// Convenience function for tests. +std::unique_ptr<object::ObjectFile> +yaml2ObjectFile(SmallVectorImpl<char> &Storage, StringRef Yaml, + ErrorHandler ErrHandler); + +} // namespace yaml +} // namespace llvm + +#endif diff --git a/include/llvm/Pass.h b/include/llvm/Pass.h index 329f7eaba73d..1d53ae32cf37 100644 --- a/include/llvm/Pass.h +++ b/include/llvm/Pass.h @@ -306,6 +306,9 @@ protected: }; //===----------------------------------------------------------------------===// +/// Deprecated - do not create new passes as BasicBlockPasses. Use FunctionPass +/// with a loop over the BasicBlocks instead. +// /// BasicBlockPass class - This class is used to implement most local /// optimizations. Optimizations should subclass this class if they /// meet the following constraints: @@ -338,6 +341,8 @@ public: /// do any post processing needed after all passes have run. virtual bool doFinalization(Function &); + void preparePassManager(PMStack &PMS) override; + void assignPassManager(PMStack &PMS, PassManagerType T) override; /// Return what kind of Pass Manager can manage this pass. diff --git a/include/llvm/Passes/PassBuilder.h b/include/llvm/Passes/PassBuilder.h index 5e6660599f93..f73e4b42dd4b 100644 --- a/include/llvm/Passes/PassBuilder.h +++ b/include/llvm/Passes/PassBuilder.h @@ -629,6 +629,12 @@ public: TopLevelPipelineParsingCallbacks.push_back(C); } + /// Add PGOInstrumenation passes for O0 only. + void addPGOInstrPassesForO0(ModulePassManager &MPM, bool DebugLogging, + bool RunProfileGen, bool IsCS, + std::string ProfileFile, + std::string ProfileRemappingFile); + private: static Optional<std::vector<PipelineElement>> parsePipelineText(StringRef Text); @@ -660,7 +666,6 @@ private: OptimizationLevel Level, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile); - void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); // Extension Point callbacks diff --git a/include/llvm/ProfileData/Coverage/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h index 11758ac4cf2f..0dd0c7ec8065 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -301,7 +301,12 @@ public: struct FunctionRecord { /// Raw function name. std::string Name; - /// Associated files. + /// Mapping from FileID (i.e. vector index) to filename. Used to support + /// macro expansions within a function in which the macro and function are + /// defined in separate files. + /// + /// TODO: Uniquing filenames across all function records may be a performance + /// optimization. std::vector<std::string> Filenames; /// Regions in the function along with their counts. std::vector<CountedRegion> CountedRegions; @@ -508,6 +513,7 @@ public: class CoverageMapping { DenseMap<size_t, DenseSet<size_t>> RecordProvenance; std::vector<FunctionRecord> Functions; + DenseMap<size_t, SmallVector<unsigned, 0>> FilenameHash2RecordIndices; std::vector<std::pair<std::string, uint64_t>> FuncHashMismatches; CoverageMapping() = default; @@ -516,6 +522,13 @@ class CoverageMapping { Error loadFunctionRecord(const CoverageMappingRecord &Record, IndexedInstrProfReader &ProfileReader); + /// Look up the indices for function records which are at least partially + /// defined in the specified file. This is guaranteed to return a superset of + /// such records: extra records not in the file may be included if there is + /// a hash collision on the filename. Clients must be robust to collisions. + ArrayRef<unsigned> + getImpreciseRecordIndicesForFilename(StringRef Filename) const; + public: CoverageMapping(const CoverageMapping &) = delete; CoverageMapping &operator=(const CoverageMapping &) = delete; @@ -527,6 +540,7 @@ public: /// Load the coverage mapping from the given object files and profile. If /// \p Arches is non-empty, it must specify an architecture for each object. + /// Ignores non-instrumented object files unless all are not instrumented. static Expected<std::unique_ptr<CoverageMapping>> load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename, ArrayRef<StringRef> Arches = None); diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h index 5f88cacdfcbb..6fcd8a09a494 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h @@ -30,8 +30,7 @@ class CoverageFilenamesSectionWriter { ArrayRef<StringRef> Filenames; public: - CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames) - : Filenames(Filenames) {} + CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames); /// Write encoded filenames to the given output stream. void write(raw_ostream &OS); diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index c7d764ade30d..c26f76949992 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -93,10 +93,6 @@ inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } /// Return the name of value profile node array variables: inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } -/// Return the name prefix of the COMDAT group for instrumentation variables -/// associated with a COMDAT function. -inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } - /// Return the name of the variable holding the strings (possibly compressed) /// of all function's PGO names. inline StringRef getInstrProfNamesVarName() { @@ -634,8 +630,8 @@ struct OverlapStats { FuncHash = Hash; } - Error accumuateCounts(const std::string &BaseFilename, - const std::string &TestFilename, bool IsCS); + Error accumulateCounts(const std::string &BaseFilename, + const std::string &TestFilename, bool IsCS); void addOneMismatch(const CountSumOrPercent &MismatchFunc); void addOneUnique(const CountSumOrPercent &UniqueFunc); @@ -695,7 +691,7 @@ struct InstrProfRecord { InstrProfRecord(const InstrProfRecord &RHS) : Counts(RHS.Counts), ValueData(RHS.ValueData - ? llvm::make_unique<ValueProfData>(*RHS.ValueData) + ? std::make_unique<ValueProfData>(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { @@ -705,7 +701,7 @@ struct InstrProfRecord { return *this; } if (!ValueData) - ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData); + ValueData = std::make_unique<ValueProfData>(*RHS.ValueData); else *ValueData = *RHS.ValueData; return *this; @@ -772,7 +768,7 @@ struct InstrProfRecord { void clearValueData() { ValueData = nullptr; } /// Compute the sums of all counts and store in Sum. - void accumuateCounts(CountSumOrPercent &Sum) const; + void accumulateCounts(CountSumOrPercent &Sum) const; /// Compute the overlap b/w this IntrprofRecord and Other. void overlap(InstrProfRecord &Other, OverlapStats &Overlap, @@ -817,7 +813,7 @@ private: std::vector<InstrProfValueSiteRecord> & getOrCreateValueSitesForKind(uint32_t ValueKind) { if (!ValueData) - ValueData = llvm::make_unique<ValueProfData>(); + ValueData = std::make_unique<ValueProfData>(); switch (ValueKind) { case IPVK_IndirectCallTarget: return ValueData->IndirectCallSites; @@ -897,7 +893,7 @@ InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, return std::unique_ptr<InstrProfValueData[]>(nullptr); } - auto VD = llvm::make_unique<InstrProfValueData[]>(N); + auto VD = std::make_unique<InstrProfValueData[]>(N); TotalCount = getValueForSite(VD.get(), ValueKind, Site); return VD; diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index 73751faab88e..f5f552672bf0 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -92,7 +92,7 @@ public: virtual InstrProfSymtab &getSymtab() = 0; /// Compute the sum of counts and return in Sum. - void accumuateCounts(CountSumOrPercent &Sum, bool IsCS); + void accumulateCounts(CountSumOrPercent &Sum, bool IsCS); protected: std::unique_ptr<InstrProfSymtab> Symtab; @@ -268,8 +268,14 @@ private: return (const char *)ValueDataStart; } - const uint64_t *getCounter(IntPtrT CounterPtr) const { - ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); + /// Get the offset of \p CounterPtr from the start of the counters section of + /// the profile. The offset has units of "number of counters", i.e. increasing + /// the offset by 1 corresponds to an increase in the *byte offset* by 8. + ptrdiff_t getCounterOffset(IntPtrT CounterPtr) const { + return (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); + } + + const uint64_t *getCounter(ptrdiff_t Offset) const { return CountersStart + Offset; } diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 7fbc857b7230..55418d9d0f9c 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -18,15 +18,18 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cstdint> #include <map> +#include <set> #include <string> #include <system_error> #include <utility> @@ -49,7 +52,10 @@ enum class sampleprof_error { truncated_name_table, not_implemented, counter_overflow, - ostream_seek_unsupported + ostream_seek_unsupported, + compress_failed, + uncompress_failed, + zlib_unavailable }; inline std::error_code make_error_code(sampleprof_error E) { @@ -83,6 +89,7 @@ enum SampleProfileFormat { SPF_Text = 0x1, SPF_Compact_Binary = 0x2, SPF_GCC = 0x3, + SPF_Ext_Binary = 0x4, SPF_Binary = 0xff }; @@ -105,6 +112,61 @@ static inline StringRef getRepInFormat(StringRef Name, static inline uint64_t SPVersion() { return 103; } +// Section Type used by SampleProfileExtBinaryBaseReader and +// SampleProfileExtBinaryBaseWriter. Never change the existing +// value of enum. Only append new ones. +enum SecType { + SecInValid = 0, + SecProfSummary = 1, + SecNameTable = 2, + SecProfileSymbolList = 3, + SecFuncOffsetTable = 4, + // marker for the first type of profile. + SecFuncProfileFirst = 32, + SecLBRProfile = SecFuncProfileFirst +}; + +static inline std::string getSecName(SecType Type) { + switch (Type) { + case SecInValid: + return "InvalidSection"; + case SecProfSummary: + return "ProfileSummarySection"; + case SecNameTable: + return "NameTableSection"; + case SecProfileSymbolList: + return "ProfileSymbolListSection"; + case SecFuncOffsetTable: + return "FuncOffsetTableSection"; + case SecLBRProfile: + return "LBRProfileSection"; + } + llvm_unreachable("A SecType has no name for output"); +} + +// Entry type of section header table used by SampleProfileExtBinaryBaseReader +// and SampleProfileExtBinaryBaseWriter. +struct SecHdrTableEntry { + SecType Type; + uint64_t Flags; + uint64_t Offset; + uint64_t Size; +}; + +enum SecFlags { SecFlagInValid = 0, SecFlagCompress = (1 << 0) }; + +static inline void addSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) { + Entry.Flags |= Flags; +} + +static inline void removeSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) { + Entry.Flags &= ~Flags; +} + +static inline bool hasSecFlag(SecHdrTableEntry &Entry, SecFlags Flag) { + return Entry.Flags & Flag; +} + /// Represents the relative location of an instruction. /// /// Instruction locations are specified by the line offset from the @@ -143,8 +205,18 @@ raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); /// will be a list of one or more functions. class SampleRecord { public: - using CallTargetMap = StringMap<uint64_t>; + using CallTarget = std::pair<StringRef, uint64_t>; + struct CallTargetComparator { + bool operator()(const CallTarget &LHS, const CallTarget &RHS) const { + if (LHS.second != RHS.second) + return LHS.second > RHS.second; + + return LHS.first < RHS.first; + } + }; + using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>; + using CallTargetMap = StringMap<uint64_t>; SampleRecord() = default; /// Increment the number of samples for this record by \p S. @@ -179,6 +251,18 @@ public: uint64_t getSamples() const { return NumSamples; } const CallTargetMap &getCallTargets() const { return CallTargets; } + const SortedCallTargetSet getSortedCallTargets() const { + return SortCallTargets(CallTargets); + } + + /// Sort call targets in descending order of call frequency. + static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) { + SortedCallTargetSet SortedTargets; + for (const auto &I : Targets) { + SortedTargets.emplace(I.first(), I.second); + } + return SortedTargets; + } /// Merge the samples in \p Other into this record. /// Optionally scale sample counts by \p Weight. @@ -205,7 +289,7 @@ class FunctionSamples; using BodySampleMap = std::map<LineLocation, SampleRecord>; // NOTE: Using a StringMap here makes parsed profiles consume around 17% more // memory, which is *very* significant for large profiles. -using FunctionSamplesMap = std::map<std::string, FunctionSamples>; +using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>; using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>; /// Representation of the samples collected for a function. @@ -447,11 +531,10 @@ public: StringRef getNameInModule(StringRef Name, const Module *M) const { if (Format != SPF_Compact_Binary) return Name; - // Expect CurrentModule to be initialized by GUIDToFuncNameMapper. - if (M != CurrentModule) - llvm_unreachable("Input Module should be the same as CurrentModule"); - auto iter = GUIDToFuncNameMap.find(std::stoull(Name.data())); - if (iter == GUIDToFuncNameMap.end()) + + assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first"); + auto iter = GUIDToFuncNameMap->find(std::stoull(Name.data())); + if (iter == GUIDToFuncNameMap->end()) return StringRef(); return iter->second; } @@ -472,42 +555,10 @@ public: const FunctionSamples *findFunctionSamples(const DILocation *DIL) const; static SampleProfileFormat Format; - /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for - /// all the function symbols defined or declared in CurrentModule. - static DenseMap<uint64_t, StringRef> GUIDToFuncNameMap; - static Module *CurrentModule; - - class GUIDToFuncNameMapper { - public: - GUIDToFuncNameMapper(Module &M) { - if (Format != SPF_Compact_Binary) - return; - - for (const auto &F : M) { - StringRef OrigName = F.getName(); - GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName}); - /// Local to global var promotion used by optimization like thinlto - /// will rename the var and add suffix like ".llvm.xxx" to the - /// original local name. In sample profile, the suffixes of function - /// names are all stripped. Since it is possible that the mapper is - /// built in post-thin-link phase and var promotion has been done, - /// we need to add the substring of function name without the suffix - /// into the GUIDToFuncNameMap. - StringRef CanonName = getCanonicalFnName(F); - if (CanonName != OrigName) - GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName}); - } - CurrentModule = &M; - } - - ~GUIDToFuncNameMapper() { - if (Format != SPF_Compact_Binary) - return; - GUIDToFuncNameMap.clear(); - CurrentModule = nullptr; - } - }; + /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for + /// all the function symbols defined or declared in current module. + DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr; // Assume the input \p Name is a name coming from FunctionSamples itself. // If the format is SPF_Compact_Binary, the name is already a GUID and we @@ -583,6 +634,47 @@ private: SamplesWithLocList V; }; +/// ProfileSymbolList records the list of function symbols shown up +/// in the binary used to generate the profile. It is useful to +/// to discriminate a function being so cold as not to shown up +/// in the profile and a function newly added. +class ProfileSymbolList { +public: + /// copy indicates whether we need to copy the underlying memory + /// for the input Name. + void add(StringRef Name, bool copy = false) { + if (!copy) { + Syms.insert(Name); + return; + } + Syms.insert(Name.copy(Allocator)); + } + + bool contains(StringRef Name) { return Syms.count(Name); } + + void merge(const ProfileSymbolList &List) { + for (auto Sym : List.Syms) + add(Sym, true); + } + + unsigned size() { return Syms.size(); } + + void setToCompress(bool TC) { ToCompress = TC; } + bool toCompress() { return ToCompress; } + + std::error_code read(const uint8_t *Data, uint64_t ListSize); + std::error_code write(raw_ostream &OS); + void dump(raw_ostream &OS = dbgs()) const; + +private: + // Determine whether or not to compress the symbol list when + // writing it into profile. The variable is unused when the symbol + // list is read from an existing profile. + bool ToCompress = false; + DenseSet<StringRef> Syms; + BumpPtrAllocator Allocator; +}; + } // end namespace sampleprof } // end namespace llvm diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 969cdea859c9..5a5d4cfde224 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -235,6 +235,62 @@ class raw_ostream; namespace sampleprof { +class SampleProfileReader; + +/// SampleProfileReaderItaniumRemapper remaps the profile data from a +/// sample profile data reader, by applying a provided set of equivalences +/// between components of the symbol names in the profile. +class SampleProfileReaderItaniumRemapper { +public: + SampleProfileReaderItaniumRemapper(std::unique_ptr<MemoryBuffer> B, + std::unique_ptr<SymbolRemappingReader> SRR, + SampleProfileReader &R) + : Buffer(std::move(B)), Remappings(std::move(SRR)), Reader(R) { + assert(Remappings && "Remappings cannot be nullptr"); + } + + /// Create a remapper from the given remapping file. The remapper will + /// be used for profile read in by Reader. + static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> + create(const std::string Filename, SampleProfileReader &Reader, + LLVMContext &C); + + /// Create a remapper from the given Buffer. The remapper will + /// be used for profile read in by Reader. + static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> + create(std::unique_ptr<MemoryBuffer> &B, SampleProfileReader &Reader, + LLVMContext &C); + + /// Apply remappings to the profile read by Reader. + void applyRemapping(LLVMContext &Ctx); + + bool hasApplied() { return RemappingApplied; } + + /// Insert function name into remapper. + void insert(StringRef FunctionName) { Remappings->insert(FunctionName); } + + /// Query whether there is equivalent in the remapper which has been + /// inserted. + bool exist(StringRef FunctionName) { + return Remappings->lookup(FunctionName); + } + + /// Return the samples collected for function \p F if remapper knows + /// it is present in SampleMap. + FunctionSamples *getSamplesFor(StringRef FunctionName); + +private: + // The buffer holding the content read from remapping file. + std::unique_ptr<MemoryBuffer> Buffer; + std::unique_ptr<SymbolRemappingReader> Remappings; + DenseMap<SymbolRemappingReader::Key, FunctionSamples *> SampleMap; + // The Reader the remapper is servicing. + SampleProfileReader &Reader; + // Indicate whether remapping has been applied to the profile read + // by Reader -- by calling applyRemapping. + bool RemappingApplied = false; +}; + /// Sample-based profile reader. /// /// Each profile contains sample counts for all the functions @@ -273,13 +329,22 @@ public: /// Read and validate the file header. virtual std::error_code readHeader() = 0; - /// Read sample profiles from the associated file. - virtual std::error_code read() = 0; + /// The interface to read sample profiles from the associated file. + std::error_code read() { + if (std::error_code EC = readImpl()) + return EC; + if (Remapper) + Remapper->applyRemapping(Ctx); + return sampleprof_error::success; + } + + /// The implementaion to read sample profiles from the associated file. + virtual std::error_code readImpl() = 0; /// Print the profile for \p FName on stream \p OS. void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); - virtual void collectFuncsToUse(const Module &M) {} + virtual void collectFuncsFrom(const Module &M) {} /// Print all the profiles on stream \p OS. void dump(raw_ostream &OS = dbgs()); @@ -295,6 +360,10 @@ public: /// Return the samples collected for function \p F. virtual FunctionSamples *getSamplesFor(StringRef Fname) { + if (Remapper) { + if (auto FS = Remapper->getSamplesFor(Fname)) + return FS; + } std::string FGUID; Fname = getRepInFormat(Fname, getFormat(), FGUID); auto It = Profiles.find(Fname); @@ -313,18 +382,33 @@ public: } /// Create a sample profile reader appropriate to the file format. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(const Twine &Filename, LLVMContext &C); + create(const std::string Filename, LLVMContext &C, + const std::string RemapFilename = ""); /// Create a sample profile reader from the supplied memory buffer. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C); + create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, + const std::string RemapFilename = ""); /// Return the profile summary. - ProfileSummary &getSummary() { return *(Summary.get()); } + ProfileSummary &getSummary() const { return *(Summary.get()); } + + MemoryBuffer *getBuffer() const { return Buffer.get(); } /// \brief Return the profile format. - SampleProfileFormat getFormat() { return Format; } + SampleProfileFormat getFormat() const { return Format; } + + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() { + return nullptr; + }; + + /// It includes all the names that have samples either in outline instance + /// or inline instance. + virtual std::vector<StringRef> *getNameTable() { return nullptr; } + virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; protected: /// Map every function to its associated profile. @@ -352,6 +436,8 @@ protected: /// Compute summary for this profile. void computeSummary(); + std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; @@ -365,7 +451,7 @@ public: std::error_code readHeader() override { return sampleprof_error::success; } /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -381,7 +467,11 @@ public: virtual std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; + + /// It includes all the names that have samples either in outline instance + /// or inline instance. + virtual std::vector<StringRef> *getNameTable() override { return &NameTable; } protected: /// Read a numeric value of type T from the profile. @@ -411,46 +501,134 @@ protected: bool at_eof() const { return Data >= End; } /// Read the next function profile instance. - std::error_code readFuncProfile(); + std::error_code readFuncProfile(const uint8_t *Start); /// Read the contents of the given profile instance. std::error_code readProfile(FunctionSamples &FProfile); + /// Read the contents of Magic number and Version number. + std::error_code readMagicIdent(); + + /// Read profile summary. + std::error_code readSummary(); + + /// Read the whole name table. + virtual std::error_code readNameTable(); + /// Points to the current location in the buffer. const uint8_t *Data = nullptr; /// Points to the end of the buffer. const uint8_t *End = nullptr; + /// Function name table. + std::vector<StringRef> NameTable; + + /// Read a string indirectly via the name table. + virtual ErrorOr<StringRef> readStringFromTable(); + private: std::error_code readSummaryEntry(std::vector<ProfileSummaryEntry> &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; +}; - /// Read profile summary. - std::error_code readSummary(); +class SampleProfileReaderRawBinary : public SampleProfileReaderBinary { +private: + virtual std::error_code verifySPMagic(uint64_t Magic) override; - /// Read the whole name table. - virtual std::error_code readNameTable() = 0; +public: + SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, + SampleProfileFormat Format = SPF_Binary) + : SampleProfileReaderBinary(std::move(B), C, Format) {} - /// Read a string indirectly via the name table. - virtual ErrorOr<StringRef> readStringFromTable() = 0; + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); }; -class SampleProfileReaderRawBinary : public SampleProfileReaderBinary { +/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase defines +/// the basic structure of the extensible binary format. +/// The format is organized in sections except the magic and version number +/// at the beginning. There is a section table before all the sections, and +/// each entry in the table describes the entry type, start, size and +/// attributes. The format in each section is defined by the section itself. +/// +/// It is easy to add a new section while maintaining the backward +/// compatibility of the profile. Nothing extra needs to be done. If we want +/// to extend an existing section, like add cache misses information in +/// addition to the sample count in the profile body, we can add a new section +/// with the extension and retire the existing section, and we could choose +/// to keep the parser of the old section if we want the reader to be able +/// to read both new and old format profile. +/// +/// SampleProfileReaderExtBinary/SampleProfileWriterExtBinary define the +/// commonly used sections of a profile in extensible binary format. It is +/// possible to define other types of profile inherited from +/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase. +class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary { +private: + std::error_code decompressSection(const uint8_t *SecStart, + const uint64_t SecSize, + const uint8_t *&DecompressBuf, + uint64_t &DecompressBufSize); + + BumpPtrAllocator Allocator; + +protected: + std::vector<SecHdrTableEntry> SecHdrTable; + std::unique_ptr<ProfileSymbolList> ProfSymList; + std::error_code readSecHdrTableEntry(); + std::error_code readSecHdrTable(); + virtual std::error_code readHeader() override; + virtual std::error_code verifySPMagic(uint64_t Magic) override = 0; + virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, + SecType Type) = 0; + +public: + SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B, + LLVMContext &C, SampleProfileFormat Format) + : SampleProfileReaderBinary(std::move(B), C, Format) {} + + /// Read sample profiles in extensible format from the associated file. + std::error_code readImpl() override; + + /// Get the total size of all \p Type sections. + uint64_t getSectionSize(SecType Type); + /// Get the total size of header and all sections. + uint64_t getFileSize(); + virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override; +}; + +class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { private: - /// Function name table. - std::vector<StringRef> NameTable; virtual std::error_code verifySPMagic(uint64_t Magic) override; - virtual std::error_code readNameTable() override; - /// Read a string indirectly via the name table. - virtual ErrorOr<StringRef> readStringFromTable() override; + virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, + SecType Type) override; + std::error_code readProfileSymbolList(); + std::error_code readFuncOffsetTable(); + std::error_code readFuncProfiles(); + + /// The table mapping from function name to the offset of its FunctionSample + /// towards file start. + DenseMap<StringRef, uint64_t> FuncOffsetTable; + /// The set containing the functions to use when compiling a module. + DenseSet<StringRef> FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; public: - SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C) - : SampleProfileReaderBinary(std::move(B), C, SPF_Binary) {} + SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, + SampleProfileFormat Format = SPF_Ext_Binary) + : SampleProfileReaderExtBinaryBase(std::move(B), C, Format) {} /// \brief Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); + + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override { + return std::move(ProfSymList); + }; + + /// Collect functions with definitions in Module \p M. + void collectFuncsFrom(const Module &M) override; }; class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { @@ -462,6 +640,8 @@ private: DenseMap<StringRef, uint64_t> FuncOffsetTable; /// The set containing the functions to use when compiling a module. DenseSet<StringRef> FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; virtual std::error_code verifySPMagic(uint64_t Magic) override; virtual std::error_code readNameTable() override; /// Read a string indirectly via the name table. @@ -478,10 +658,10 @@ public: static bool hasFormat(const MemoryBuffer &Buffer); /// Read samples only for functions to use. - std::error_code read() override; + std::error_code readImpl() override; /// Collect functions to be used when compiling Module \p M. - void collectFuncsToUse(const Module &M) override; + void collectFuncsFrom(const Module &M) override; }; using InlineCallStack = SmallVector<FunctionSamples *, 10>; @@ -509,7 +689,7 @@ public: std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -537,44 +717,6 @@ protected: static const uint32_t GCOVTagAFDOFunction = 0xac000000; }; -/// A profile data reader proxy that remaps the profile data from another -/// sample profile data reader, by applying a provided set of equivalences -/// between components of the symbol names in the profile. -class SampleProfileReaderItaniumRemapper : public SampleProfileReader { -public: - SampleProfileReaderItaniumRemapper( - std::unique_ptr<MemoryBuffer> B, LLVMContext &C, - std::unique_ptr<SampleProfileReader> Underlying) - : SampleProfileReader(std::move(B), C, Underlying->getFormat()) { - Profiles = std::move(Underlying->getProfiles()); - Summary = takeSummary(*Underlying); - // Keep the underlying reader alive; the profile data may contain - // StringRefs referencing names in its name table. - UnderlyingReader = std::move(Underlying); - } - - /// Create a remapped sample profile from the given remapping file and - /// underlying samples. - static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(const Twine &Filename, LLVMContext &C, - std::unique_ptr<SampleProfileReader> Underlying); - - /// Read and validate the file header. - std::error_code readHeader() override { return sampleprof_error::success; } - - /// Read remapping file and apply it to the sample profile. - std::error_code read() override; - - /// Return the samples collected for function \p F. - FunctionSamples *getSamplesFor(StringRef FunctionName) override; - using SampleProfileReader::getSamplesFor; - -private: - SymbolRemappingReader Remappings; - DenseMap<SymbolRemappingReader::Key, FunctionSamples*> SampleMap; - std::unique_ptr<SampleProfileReader> UnderlyingReader; -}; - } // end namespace sampleprof } // end namespace llvm diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 81e6e3ab0b4a..cc951594c9e2 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -36,7 +36,7 @@ public: /// Write sample profiles in \p S. /// /// \returns status code of the file update operation. - virtual std::error_code write(const FunctionSamples &S) = 0; + virtual std::error_code writeSample(const FunctionSamples &S) = 0; /// Write all the sample profiles in the given map of samples. /// @@ -56,6 +56,8 @@ public: static ErrorOr<std::unique_ptr<SampleProfileWriter>> create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); + virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} + protected: SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) : OutputStream(std::move(OS)) {} @@ -64,6 +66,10 @@ protected: virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) = 0; + // Write function profiles to the profile file. + virtual std::error_code + writeFuncProfiles(const StringMap<FunctionSamples> &ProfileMap); + /// Output stream where to emit the profile to. std::unique_ptr<raw_ostream> OutputStream; @@ -72,12 +78,15 @@ protected: /// Compute summary for this profile. void computeSummary(const StringMap<FunctionSamples> &ProfileMap); + + /// Profile format. + SampleProfileFormat Format; }; /// Sample-based profile writer (text format). class SampleProfileWriterText : public SampleProfileWriter { public: - std::error_code write(const FunctionSamples &S) override; + std::error_code writeSample(const FunctionSamples &S) override; protected: SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) @@ -102,13 +111,14 @@ private: /// Sample-based profile writer (binary format). class SampleProfileWriterBinary : public SampleProfileWriter { public: - virtual std::error_code write(const FunctionSamples &S) override; SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) : SampleProfileWriter(OS) {} + virtual std::error_code writeSample(const FunctionSamples &S) override; + protected: - virtual std::error_code writeNameTable() = 0; - virtual std::error_code writeMagicIdent() = 0; + virtual std::error_code writeMagicIdent(SampleProfileFormat Format); + virtual std::error_code writeNameTable(); virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; std::error_code writeSummary(); @@ -118,10 +128,10 @@ protected: MapVector<StringRef, uint32_t> NameTable; -private: void addName(StringRef FName); void addNames(const FunctionSamples &S); +private: friend ErrorOr<std::unique_ptr<SampleProfileWriter>> SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); @@ -129,10 +139,99 @@ private: class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; +}; + +class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { + using SampleProfileWriterBinary::SampleProfileWriterBinary; +public: + virtual std::error_code + write(const StringMap<FunctionSamples> &ProfileMap) override; + + void setToCompressAllSections(); + void setToCompressSection(SecType Type); protected: - virtual std::error_code writeNameTable() override; - virtual std::error_code writeMagicIdent() override; + uint64_t markSectionStart(SecType Type); + std::error_code addNewSection(SecType Sec, uint64_t SectionStart); + virtual void initSectionHdrLayout() = 0; + virtual std::error_code + writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0; + + // Specifiy the order of sections in section header table. Note + // the order of sections in the profile may be different that the + // order in SectionHdrLayout. sample Reader will follow the order + // in SectionHdrLayout to read each section. + SmallVector<SecHdrTableEntry, 8> SectionHdrLayout; + +private: + void allocSecHdrTable(); + std::error_code writeSecHdrTable(); + virtual std::error_code + writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; + void addSectionFlags(SecType Type, SecFlags Flags); + SecHdrTableEntry &getEntryInLayout(SecType Type); + std::error_code compressAndOutput(); + + // We will swap the raw_ostream held by LocalBufStream and that + // held by OutputStream if we try to add a section which needs + // compression. After the swap, all the data written to output + // will be temporarily buffered into the underlying raw_string_ostream + // originally held by LocalBufStream. After the data writing for the + // section is completed, compress the data in the local buffer, + // swap the raw_ostream back and write the compressed data to the + // real output. + std::unique_ptr<raw_ostream> LocalBufStream; + // The location where the output stream starts. + uint64_t FileStart; + // The location in the output stream where the SecHdrTable should be + // written to. + uint64_t SecHdrTableOffset; + // Initial Section Flags setting. + std::vector<SecHdrTableEntry> SecHdrTable; +}; + +class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { +public: + SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) + : SampleProfileWriterExtBinaryBase(OS) { + initSectionHdrLayout(); + } + + virtual std::error_code writeSample(const FunctionSamples &S) override; + virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { + ProfSymList = PSL; + }; + +private: + virtual void initSectionHdrLayout() override { + // Note that SecFuncOffsetTable section is written after SecLBRProfile + // in the profile, but is put before SecLBRProfile in SectionHdrLayout. + // + // This is because sample reader follows the order of SectionHdrLayout to + // read each section, to read function profiles on demand sample reader + // need to get the offset of each function profile first. + // + // SecFuncOffsetTable section is written after SecLBRProfile in the + // profile because FuncOffsetTable needs to be populated while section + // SecLBRProfile is written. + SectionHdrLayout = {{SecProfSummary, 0, 0, 0}, + {SecNameTable, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0}}; + }; + virtual std::error_code + writeSections(const StringMap<FunctionSamples> &ProfileMap) override; + ProfileSymbolList *ProfSymList = nullptr; + + // Save the start of SecLBRProfile so we can compute the offset to the + // start of SecLBRProfile for each Function's Profile and will keep it + // in FuncOffsetTable. + uint64_t SecLBRProfileStart; + // FuncOffsetTable maps function name to its profile offset in SecLBRProfile + // section. It is used to load function profile on demand. + MapVector<StringRef, uint64_t> FuncOffsetTable; + std::error_code writeFuncOffsetTable(); }; // CompactBinary is a compact format of binary profile which both reduces @@ -169,7 +268,7 @@ class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; public: - virtual std::error_code write(const FunctionSamples &S) override; + virtual std::error_code writeSample(const FunctionSamples &S) override; virtual std::error_code write(const StringMap<FunctionSamples> &ProfileMap) override; @@ -181,7 +280,6 @@ protected: /// towards profile start. uint64_t TableOffset; virtual std::error_code writeNameTable() override; - virtual std::error_code writeMagicIdent() override; virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; std::error_code writeFuncOffsetTable(); diff --git a/include/llvm/Remarks/BitstreamRemarkContainer.h b/include/llvm/Remarks/BitstreamRemarkContainer.h new file mode 100644 index 000000000000..a2282fca04ab --- /dev/null +++ b/include/llvm/Remarks/BitstreamRemarkContainer.h @@ -0,0 +1,106 @@ +//===-- BitstreamRemarkContainer.h - Container for remarks --------------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides declarations for things used in the various types of +// remark containers. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_CONTAINER_H +#define LLVM_REMARKS_REMARK_CONTAINER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitstream/BitCodes.h" +#include <cstdint> + +namespace llvm { +namespace remarks { + +/// The current version of the remark container. +/// Note: this is different from the version of the remark entry. +constexpr uint64_t CurrentContainerVersion = 0; +/// The magic number used for identifying remark blocks. +constexpr StringLiteral ContainerMagic("RMRK"); + +/// Type of the remark container. +/// The remark container has two modes: +/// * separate: the metadata is separate from the remarks and points to the +/// auxiliary file that contains the remarks. +/// * standalone: the metadata and the remarks are emitted together. +enum class BitstreamRemarkContainerType { + /// The metadata emitted separately. + /// This will contain the following: + /// * Container version and type + /// * String table + /// * External file + SeparateRemarksMeta, + /// The remarks emitted separately. + /// This will contain the following: + /// * Container version and type + /// * Remark version + SeparateRemarksFile, + /// Everything is emitted together. + /// This will contain the following: + /// * Container version and type + /// * Remark version + /// * String table + Standalone, + First = SeparateRemarksMeta, + Last = Standalone, +}; + +/// The possible blocks that will be encountered in a bitstream remark +/// container. +enum BlockIDs { + /// The metadata block is mandatory. It should always come after the + /// BLOCKINFO_BLOCK, and contains metadata that should be used when parsing + /// REMARK_BLOCKs. + /// There should always be only one META_BLOCK. + META_BLOCK_ID = bitc::FIRST_APPLICATION_BLOCKID, + /// One remark entry is represented using a REMARK_BLOCK. There can be + /// multiple REMARK_BLOCKs in the same file. + REMARK_BLOCK_ID +}; + +constexpr StringRef MetaBlockName = StringRef("Meta", 4); +constexpr StringRef RemarkBlockName = StringRef("Remark", 6); + +/// The possible records that can be encountered in the previously described +/// blocks. +enum RecordIDs { + // Meta block records. + RECORD_META_CONTAINER_INFO = 1, + RECORD_META_REMARK_VERSION, + RECORD_META_STRTAB, + RECORD_META_EXTERNAL_FILE, + // Remark block records. + RECORD_REMARK_HEADER, + RECORD_REMARK_DEBUG_LOC, + RECORD_REMARK_HOTNESS, + RECORD_REMARK_ARG_WITH_DEBUGLOC, + RECORD_REMARK_ARG_WITHOUT_DEBUGLOC, + // Helpers. + RECORD_FIRST = RECORD_META_CONTAINER_INFO, + RECORD_LAST = RECORD_REMARK_ARG_WITHOUT_DEBUGLOC +}; + +constexpr StringRef MetaContainerInfoName = StringRef("Container info", 14); +constexpr StringRef MetaRemarkVersionName = StringRef("Remark version", 14); +constexpr StringRef MetaStrTabName = StringRef("String table", 12); +constexpr StringRef MetaExternalFileName = StringRef("External File", 13); +constexpr StringRef RemarkHeaderName = StringRef("Remark header", 13); +constexpr StringRef RemarkDebugLocName = StringRef("Remark debug location", 21); +constexpr StringRef RemarkHotnessName = StringRef("Remark hotness", 14); +constexpr StringRef RemarkArgWithDebugLocName = + StringRef("Argument with debug location", 28); +constexpr StringRef RemarkArgWithoutDebugLocName = StringRef("Argument", 8); + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_CONTAINER_H */ diff --git a/include/llvm/Remarks/BitstreamRemarkParser.h b/include/llvm/Remarks/BitstreamRemarkParser.h new file mode 100644 index 000000000000..7ebd731693b2 --- /dev/null +++ b/include/llvm/Remarks/BitstreamRemarkParser.h @@ -0,0 +1,116 @@ +//===-- BitstreamRemarkParser.h - Bitstream parser --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an implementation of the remark parser using the LLVM +// Bitstream format. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_BITSTREAM_REMARK_PARSER_H +#define LLVM_REMARKS_BITSTREAM_REMARK_PARSER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/Remarks/BitstreamRemarkContainer.h" +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkParser.h" +#include "llvm/Support/Error.h" +#include <array> + +namespace llvm { +namespace remarks { + +/// Helper to parse a META_BLOCK for a bitstream remark container. +struct BitstreamMetaParserHelper { + /// The Bitstream reader. + BitstreamCursor &Stream; + /// Reference to the storage for the block info. + BitstreamBlockInfo &BlockInfo; + /// The parsed content: depending on the container type, some fields might be + /// empty. + Optional<uint64_t> ContainerVersion; + Optional<uint8_t> ContainerType; + Optional<StringRef> StrTabBuf; + Optional<StringRef> ExternalFilePath; + Optional<uint64_t> RemarkVersion; + + /// Continue parsing with \p Stream. \p Stream is expected to contain a + /// ENTER_SUBBLOCK to the META_BLOCK at the current position. + /// \p Stream is expected to have a BLOCKINFO_BLOCK set. + BitstreamMetaParserHelper(BitstreamCursor &Stream, + BitstreamBlockInfo &BlockInfo); + + /// Parse the META_BLOCK and fill the available entries. + /// This helper does not check for the validity of the fields. + Error parse(); +}; + +/// Helper to parse a REMARK_BLOCK for a bitstream remark container. +struct BitstreamRemarkParserHelper { + /// The Bitstream reader. + BitstreamCursor &Stream; + /// The parsed content: depending on the remark, some fields might be empty. + Optional<uint8_t> Type; + Optional<uint64_t> RemarkNameIdx; + Optional<uint64_t> PassNameIdx; + Optional<uint64_t> FunctionNameIdx; + Optional<uint64_t> SourceFileNameIdx; + Optional<uint32_t> SourceLine; + Optional<uint32_t> SourceColumn; + Optional<uint64_t> Hotness; + struct Argument { + Optional<uint64_t> KeyIdx; + Optional<uint64_t> ValueIdx; + Optional<uint64_t> SourceFileNameIdx; + Optional<uint32_t> SourceLine; + Optional<uint32_t> SourceColumn; + }; + Optional<ArrayRef<Argument>> Args; + /// Avoid re-allocating a vector every time. + SmallVector<Argument, 8> TmpArgs; + + /// Continue parsing with \p Stream. \p Stream is expected to contain a + /// ENTER_SUBBLOCK to the REMARK_BLOCK at the current position. + /// \p Stream is expected to have a BLOCKINFO_BLOCK set and to have already + /// parsed the META_BLOCK. + BitstreamRemarkParserHelper(BitstreamCursor &Stream); + + /// Parse the REMARK_BLOCK and fill the available entries. + /// This helper does not check for the validity of the fields. + Error parse(); +}; + +/// Helper to parse any bitstream remark container. +struct BitstreamParserHelper { + /// The Bitstream reader. + BitstreamCursor Stream; + /// The block info block. + BitstreamBlockInfo BlockInfo; + /// Start parsing at \p Buffer. + BitstreamParserHelper(StringRef Buffer); + /// Parse the magic number. + Expected<std::array<char, 4>> parseMagic(); + /// Parse the block info block containing all the abbrevs. + /// This needs to be called before calling any other parsing function. + Error parseBlockInfoBlock(); + /// Return true if the next block is a META_BLOCK. This function does not move + /// the cursor. + Expected<bool> isMetaBlock(); + /// Return true if the next block is a REMARK_BLOCK. This function does not + /// move the cursor. + Expected<bool> isRemarkBlock(); + /// Return true if the parser reached the end of the stream. + bool atEndOfStream() { return Stream.AtEndOfStream(); } + /// Jump to the end of the stream, skipping everything. + void skipToEnd() { return Stream.skipToEnd(); } +}; + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_BITSTREAM_REMARK_PARSER_H */ diff --git a/include/llvm/Remarks/BitstreamRemarkSerializer.h b/include/llvm/Remarks/BitstreamRemarkSerializer.h new file mode 100644 index 000000000000..62a175a1db0b --- /dev/null +++ b/include/llvm/Remarks/BitstreamRemarkSerializer.h @@ -0,0 +1,196 @@ +//===-- BitstreamRemarkSerializer.h - Bitstream serializer ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an implementation of the serializer using the LLVM +// Bitstream format. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_BITSTREAM_REMARK_SERIALIZER_H +#define LLVM_REMARKS_BITSTREAM_REMARK_SERIALIZER_H + +#include "llvm/Bitstream/BitstreamWriter.h" +#include "llvm/Remarks/BitstreamRemarkContainer.h" +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace remarks { + +/// Serialize the remarks to LLVM bitstream. +/// This class provides ways to emit remarks in the LLVM bitstream format and +/// its associated metadata. +/// +/// * The separate model: +/// Separate meta: | Container info +/// | String table +/// | External file +/// +/// Separate remarks: | Container info +/// | Remark version +/// | Remark0 +/// | Remark1 +/// | Remark2 +/// | ... +/// +/// * The standalone model: | Container info +/// | String table +/// | Remark version +/// | Remark0 +/// | Remark1 +/// | Remark2 +/// | ... +/// +struct BitstreamRemarkSerializerHelper { + /// Buffer used for encoding the bitstream before writing it to the final + /// stream. + SmallVector<char, 1024> Encoded; + /// Buffer used to construct records and pass to the bitstream writer. + SmallVector<uint64_t, 64> R; + /// The Bitstream writer. + BitstreamWriter Bitstream; + /// The type of the container we are serializing. + BitstreamRemarkContainerType ContainerType; + + /// Abbrev IDs initialized in the block info block. + /// Note: depending on the container type, some IDs might be uninitialized. + /// Warning: When adding more abbrev IDs, make sure to update the + /// BlockCodeSize (in the call to EnterSubblock). + uint64_t RecordMetaContainerInfoAbbrevID = 0; + uint64_t RecordMetaRemarkVersionAbbrevID = 0; + uint64_t RecordMetaStrTabAbbrevID = 0; + uint64_t RecordMetaExternalFileAbbrevID = 0; + uint64_t RecordRemarkHeaderAbbrevID = 0; + uint64_t RecordRemarkDebugLocAbbrevID = 0; + uint64_t RecordRemarkHotnessAbbrevID = 0; + uint64_t RecordRemarkArgWithDebugLocAbbrevID = 0; + uint64_t RecordRemarkArgWithoutDebugLocAbbrevID = 0; + + BitstreamRemarkSerializerHelper(BitstreamRemarkContainerType ContainerType); + + // Disable copy and move: Bitstream points to Encoded, which needs special + // handling during copy/move, but moving the vectors is probably useless + // anyway. + BitstreamRemarkSerializerHelper(const BitstreamRemarkSerializerHelper &) = + delete; + BitstreamRemarkSerializerHelper & + operator=(const BitstreamRemarkSerializerHelper &) = delete; + BitstreamRemarkSerializerHelper(BitstreamRemarkSerializerHelper &&) = delete; + BitstreamRemarkSerializerHelper & + operator=(BitstreamRemarkSerializerHelper &&) = delete; + + /// Set up the necessary block info entries according to the container type. + void setupBlockInfo(); + + /// Set up the block info for the metadata block. + void setupMetaBlockInfo(); + /// The remark version in the metadata block. + void setupMetaRemarkVersion(); + void emitMetaRemarkVersion(uint64_t RemarkVersion); + /// The strtab in the metadata block. + void setupMetaStrTab(); + void emitMetaStrTab(const StringTable &StrTab); + /// The external file in the metadata block. + void setupMetaExternalFile(); + void emitMetaExternalFile(StringRef Filename); + + /// The block info for the remarks block. + void setupRemarkBlockInfo(); + + /// Emit the metadata for the remarks. + void emitMetaBlock(uint64_t ContainerVersion, + Optional<uint64_t> RemarkVersion, + Optional<const StringTable *> StrTab = None, + Optional<StringRef> Filename = None); + + /// Emit a remark block. The string table is required. + void emitRemarkBlock(const Remark &Remark, StringTable &StrTab); + /// Finalize the writing to \p OS. + void flushToStream(raw_ostream &OS); + /// Finalize the writing to a buffer. + /// The contents of the buffer remain valid for the lifetime of the object. + /// Any call to any other function in this class will invalidate the buffer. + StringRef getBuffer(); +}; + +/// Implementation of the remark serializer using LLVM bitstream. +struct BitstreamRemarkSerializer : public RemarkSerializer { + /// The file should contain: + /// 1) The block info block that describes how to read the blocks. + /// 2) The metadata block that contains various information about the remarks + /// in the file. + /// 3) A number of remark blocks. + + /// We need to set up 1) and 2) first, so that we can emit 3) after. This flag + /// is used to emit the first two blocks only once. + bool DidSetUp = false; + /// The helper to emit bitstream. + BitstreamRemarkSerializerHelper Helper; + + /// Construct a serializer that will create its own string table. + BitstreamRemarkSerializer(raw_ostream &OS, SerializerMode Mode); + /// Construct a serializer with a pre-filled string table. + BitstreamRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + StringTable StrTab); + + /// Emit a remark to the stream. This also emits the metadata associated to + /// the remarks based on the SerializerMode specified at construction. + /// This writes the serialized output to the provided stream. + void emit(const Remark &Remark) override; + /// The metadata serializer associated to this remark serializer. Based on the + /// container type of the current serializer, the container type of the + /// metadata serializer will change. + std::unique_ptr<MetaSerializer> + metaSerializer(raw_ostream &OS, + Optional<StringRef> ExternalFilename = None) override; + + static bool classof(const RemarkSerializer *S) { + return S->SerializerFormat == Format::Bitstream; + } +}; + +/// Serializer of metadata for bitstream remarks. +struct BitstreamMetaSerializer : public MetaSerializer { + /// This class can be used with [1] a pre-constructed + /// BitstreamRemarkSerializerHelper, or with [2] one that is owned by the meta + /// serializer. In case of [1], we need to be able to store a reference to the + /// object, while in case of [2] we need to store the whole object. + Optional<BitstreamRemarkSerializerHelper> TmpHelper; + /// The actual helper, that can point to \p TmpHelper or to an external helper + /// object. + BitstreamRemarkSerializerHelper *Helper = nullptr; + + Optional<const StringTable *> StrTab; + Optional<StringRef> ExternalFilename; + + /// Create a new meta serializer based on \p ContainerType. + BitstreamMetaSerializer(raw_ostream &OS, + BitstreamRemarkContainerType ContainerType, + Optional<const StringTable *> StrTab = None, + Optional<StringRef> ExternalFilename = None) + : MetaSerializer(OS), TmpHelper(None), Helper(nullptr), StrTab(StrTab), + ExternalFilename(ExternalFilename) { + TmpHelper.emplace(ContainerType); + Helper = &*TmpHelper; + } + + /// Create a new meta serializer based on a previously built \p Helper. + BitstreamMetaSerializer(raw_ostream &OS, + BitstreamRemarkSerializerHelper &Helper, + Optional<const StringTable *> StrTab = None, + Optional<StringRef> ExternalFilename = None) + : MetaSerializer(OS), TmpHelper(None), Helper(&Helper), StrTab(StrTab), + ExternalFilename(ExternalFilename) {} + + void emit() override; +}; + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_BITSTREAM_REMARK_SERIALIZER_H */ diff --git a/include/llvm/Remarks/Remark.h b/include/llvm/Remarks/Remark.h index 05d0ea60accd..1243311fb8c5 100644 --- a/include/llvm/Remarks/Remark.h +++ b/include/llvm/Remarks/Remark.h @@ -23,7 +23,8 @@ namespace llvm { namespace remarks { -constexpr uint64_t Version = 0; +/// The current version of the remark entry. +constexpr uint64_t CurrentRemarkVersion = 0; /// The debug location used to track a remark back to the source file. struct RemarkLocation { @@ -58,7 +59,8 @@ enum class Type { AnalysisFPCommute, AnalysisAliasing, Failure, - LastTypeValue = Failure + First = Unknown, + Last = Failure }; /// A remark type used for both emission and parsing. @@ -107,6 +109,36 @@ private: // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Remark, LLVMRemarkEntryRef) +/// Comparison operators for Remark objects and dependent objects. +inline bool operator==(const RemarkLocation &LHS, const RemarkLocation &RHS) { + return LHS.SourceFilePath == RHS.SourceFilePath && + LHS.SourceLine == RHS.SourceLine && + LHS.SourceColumn == RHS.SourceColumn; +} + +inline bool operator!=(const RemarkLocation &LHS, const RemarkLocation &RHS) { + return !(LHS == RHS); +} + +inline bool operator==(const Argument &LHS, const Argument &RHS) { + return LHS.Key == RHS.Key && LHS.Val == RHS.Val && LHS.Loc == RHS.Loc; +} + +inline bool operator!=(const Argument &LHS, const Argument &RHS) { + return !(LHS == RHS); +} + +inline bool operator==(const Remark &LHS, const Remark &RHS) { + return LHS.RemarkType == RHS.RemarkType && LHS.PassName == RHS.PassName && + LHS.RemarkName == RHS.RemarkName && + LHS.FunctionName == RHS.FunctionName && LHS.Loc == RHS.Loc && + LHS.Hotness == RHS.Hotness && LHS.Args == RHS.Args; +} + +inline bool operator!=(const Remark &LHS, const Remark &RHS) { + return !(LHS == RHS); +} + } // end namespace remarks } // end namespace llvm diff --git a/include/llvm/Remarks/RemarkFormat.h b/include/llvm/Remarks/RemarkFormat.h index e167d99d2517..6dd32b226099 100644 --- a/include/llvm/Remarks/RemarkFormat.h +++ b/include/llvm/Remarks/RemarkFormat.h @@ -19,10 +19,10 @@ namespace llvm { namespace remarks { -constexpr StringRef Magic("REMARKS", 7); +constexpr StringLiteral Magic("REMARKS"); /// The format used for serializing/deserializing remarks. -enum class Format { Unknown, YAML }; +enum class Format { Unknown, YAML, YAMLStrTab, Bitstream }; /// Parse and validate a string for the remark format. Expected<Format> parseFormat(StringRef FormatStr); diff --git a/include/llvm/Remarks/RemarkParser.h b/include/llvm/Remarks/RemarkParser.h index 671e1abe5ec7..d6b1fddb06ff 100644 --- a/include/llvm/Remarks/RemarkParser.h +++ b/include/llvm/Remarks/RemarkParser.h @@ -23,9 +23,6 @@ namespace llvm { namespace remarks { -struct ParserImpl; -struct ParsedStringTable; - class EndOfFileError : public ErrorInfo<EndOfFileError> { public: static char ID; @@ -39,11 +36,13 @@ public: }; /// Parser used to parse a raw buffer to remarks::Remark objects. -struct Parser { +struct RemarkParser { /// The format of the parser. Format ParserFormat; + /// Path to prepend when opening an external remark file. + std::string ExternalFilePrependPath; - Parser(Format ParserFormat) : ParserFormat(ParserFormat) {} + RemarkParser(Format ParserFormat) : ParserFormat(ParserFormat) {} /// If no error occurs, this returns a valid Remark object. /// If an error of type EndOfFileError occurs, it is safe to recover from it @@ -52,7 +51,7 @@ struct Parser { /// The pointer should never be null. virtual Expected<std::unique_ptr<Remark>> next() = 0; - virtual ~Parser() = default; + virtual ~RemarkParser() = default; }; /// In-memory representation of the string table parsed from a buffer (e.g. the @@ -60,16 +59,33 @@ struct Parser { struct ParsedStringTable { /// The buffer mapped from the section contents. StringRef Buffer; - /// Collection of offsets in the buffer for each string entry. - SmallVector<size_t, 8> Offsets; + /// This object has high changes to be std::move'd around, so don't use a + /// SmallVector for once. + std::vector<size_t> Offsets; - Expected<StringRef> operator[](size_t Index) const; ParsedStringTable(StringRef Buffer); + /// Disable copy. + ParsedStringTable(const ParsedStringTable &) = delete; + ParsedStringTable &operator=(const ParsedStringTable &) = delete; + /// Should be movable. + ParsedStringTable(ParsedStringTable &&) = default; + ParsedStringTable &operator=(ParsedStringTable &&) = default; + + size_t size() const { return Offsets.size(); } + Expected<StringRef> operator[](size_t Index) const; }; -Expected<std::unique_ptr<Parser>> +Expected<std::unique_ptr<RemarkParser>> createRemarkParser(Format ParserFormat, + StringRef Buf); + +Expected<std::unique_ptr<RemarkParser>> createRemarkParser(Format ParserFormat, StringRef Buf, - Optional<const ParsedStringTable *> StrTab = None); + ParsedStringTable StrTab); + +Expected<std::unique_ptr<RemarkParser>> +createRemarkParserFromMeta(Format ParserFormat, StringRef Buf, + Optional<ParsedStringTable> StrTab = None, + Optional<StringRef> ExternalFilePrependPath = None); } // end namespace remarks } // end namespace llvm diff --git a/include/llvm/Remarks/RemarkSerializer.h b/include/llvm/Remarks/RemarkSerializer.h index def5c2e16620..35752cd5f6fb 100644 --- a/include/llvm/Remarks/RemarkSerializer.h +++ b/include/llvm/Remarks/RemarkSerializer.h @@ -14,54 +14,74 @@ #define LLVM_REMARKS_REMARK_SERIALIZER_H #include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkStringTable.h" -#include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" namespace llvm { namespace remarks { +enum class SerializerMode { + Separate, // A mode where the metadata is serialized separately from the + // remarks. Typically, this is used when the remarks need to be + // streamed to a side file and the metadata is embedded into the + // final result of the compilation. + Standalone // A mode where everything can be retrieved in the same + // file/buffer. Typically, this is used for storing remarks for + // later use. +}; + +struct MetaSerializer; + /// This is the base class for a remark serializer. /// It includes support for using a string table while emitting. -struct Serializer { +struct RemarkSerializer { + /// The format of the serializer. + Format SerializerFormat; /// The open raw_ostream that the remark diagnostics are emitted to. raw_ostream &OS; + /// The serialization mode. + SerializerMode Mode; /// The string table containing all the unique strings used in the output. /// The table can be serialized to be consumed after the compilation. Optional<StringTable> StrTab; - Serializer(raw_ostream &OS) : OS(OS), StrTab() {} + RemarkSerializer(Format SerializerFormat, raw_ostream &OS, + SerializerMode Mode) + : SerializerFormat(SerializerFormat), OS(OS), Mode(Mode), StrTab() {} /// This is just an interface. - virtual ~Serializer() = default; + virtual ~RemarkSerializer() = default; + /// Emit a remark to the stream. virtual void emit(const Remark &Remark) = 0; + /// Return the corresponding metadata serializer. + virtual std::unique_ptr<MetaSerializer> + metaSerializer(raw_ostream &OS, + Optional<StringRef> ExternalFilename = None) = 0; }; -/// Wether the serializer should use a string table while emitting. -enum class UseStringTable { No, Yes }; - -/// Serialize the remarks to YAML. One remark entry looks like this: -/// --- !<TYPE> -/// Pass: <PASSNAME> -/// Name: <REMARKNAME> -/// DebugLoc: { File: <SOURCEFILENAME>, Line: <SOURCELINE>, -/// Column: <SOURCECOLUMN> } -/// Function: <FUNCTIONNAME> -/// Args: -/// - <KEY>: <VALUE> -/// DebugLoc: { File: <FILE>, Line: <LINE>, Column: <COL> } -/// ... -struct YAMLSerializer : public Serializer { - /// The YAML streamer. - yaml::Output YAMLOutput; +/// This is the base class for a remark metadata serializer. +struct MetaSerializer { + /// The open raw_ostream that the metadata is emitted to. + raw_ostream &OS; - YAMLSerializer(raw_ostream &OS, - UseStringTable UseStringTable = remarks::UseStringTable::No); + MetaSerializer(raw_ostream &OS) : OS(OS) {} - /// Emit a remark to the stream. - void emit(const Remark &Remark) override; + /// This is just an interface. + virtual ~MetaSerializer() = default; + virtual void emit() = 0; }; +/// Create a remark serializer. +Expected<std::unique_ptr<RemarkSerializer>> +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS); + +/// Create a remark serializer that uses a pre-filled string table. +Expected<std::unique_ptr<RemarkSerializer>> +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab); + } // end namespace remarks } // end namespace llvm diff --git a/include/llvm/Remarks/RemarkStringTable.h b/include/llvm/Remarks/RemarkStringTable.h index f9b4fdbbfb8d..4ce27ee884c8 100644 --- a/include/llvm/Remarks/RemarkStringTable.h +++ b/include/llvm/Remarks/RemarkStringTable.h @@ -18,7 +18,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Allocator.h" +#include "llvm/Remarks/Remark.h" #include <vector> namespace llvm { @@ -27,21 +27,35 @@ class raw_ostream; namespace remarks { +struct ParsedStringTable; + /// The string table used for serializing remarks. /// This table can be for example serialized in a section to be consumed after /// the compilation. struct StringTable { - /// Allocator holding all the memory used by the map. - BumpPtrAllocator Allocator; /// The string table containing all the unique strings used in the output. /// It maps a string to an unique ID. - StringMap<unsigned, BumpPtrAllocator &> StrTab; + StringMap<unsigned, BumpPtrAllocator> StrTab; /// Total size of the string table when serialized. size_t SerializedSize = 0; - StringTable() : Allocator(), StrTab(Allocator) {} + StringTable() = default; + + /// Disable copy. + StringTable(const StringTable &) = delete; + StringTable &operator=(const StringTable &) = delete; + /// Should be movable. + StringTable(StringTable &&) = default; + StringTable &operator=(StringTable &&) = default; + + /// Construct a string table from a ParsedStringTable. + StringTable(const ParsedStringTable &Other); + /// Add a string to the table. It returns an unique ID of the string. std::pair<unsigned, StringRef> add(StringRef Str); + /// Modify \p R to use strings from this string table. If the string table + /// does not contain the strings, it adds them. + void internalize(Remark &R); /// Serialize the string table to a stream. It is serialized as a little /// endian uint64 (the size of the table in bytes) followed by a sequence of /// NULL-terminated strings, where the N-th string is the string with the ID N diff --git a/include/llvm/Remarks/YAMLRemarkSerializer.h b/include/llvm/Remarks/YAMLRemarkSerializer.h new file mode 100644 index 000000000000..f1213beab15d --- /dev/null +++ b/include/llvm/Remarks/YAMLRemarkSerializer.h @@ -0,0 +1,108 @@ +//===-- YAMLRemarkSerializer.h - YAML Remark serialization ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an interface for serializing remarks to YAML. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_YAML_REMARK_SERIALIZER_H +#define LLVM_REMARKS_YAML_REMARK_SERIALIZER_H + +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Support/YAMLTraits.h" + +namespace llvm { +namespace remarks { + +/// Serialize the remarks to YAML. One remark entry looks like this: +/// --- !<TYPE> +/// Pass: <PASSNAME> +/// Name: <REMARKNAME> +/// DebugLoc: { File: <SOURCEFILENAME>, Line: <SOURCELINE>, +/// Column: <SOURCECOLUMN> } +/// Function: <FUNCTIONNAME> +/// Args: +/// - <KEY>: <VALUE> +/// DebugLoc: { File: <FILE>, Line: <LINE>, Column: <COL> } +/// ... +struct YAMLRemarkSerializer : public RemarkSerializer { + /// The YAML streamer. + yaml::Output YAMLOutput; + + YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + Optional<StringTable> StrTab = None); + + void emit(const Remark &Remark) override; + std::unique_ptr<MetaSerializer> + metaSerializer(raw_ostream &OS, + Optional<StringRef> ExternalFilename = None) override; + + static bool classof(const RemarkSerializer *S) { + return S->SerializerFormat == Format::YAML; + } + +protected: + YAMLRemarkSerializer(Format SerializerFormat, raw_ostream &OS, + SerializerMode Mode, + Optional<StringTable> StrTab = None); +}; + +struct YAMLMetaSerializer : public MetaSerializer { + Optional<StringRef> ExternalFilename; + + YAMLMetaSerializer(raw_ostream &OS, Optional<StringRef> ExternalFilename) + : MetaSerializer(OS), ExternalFilename(ExternalFilename) {} + + void emit() override; +}; + +/// Serialize the remarks to YAML using a string table. An remark entry looks +/// like the regular YAML remark but instead of string entries it's using +/// numbers that map to an index in the string table. +struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer { + /// Wether we already emitted the metadata in standalone mode. + /// This should be set to true after the first invocation of `emit`. + bool DidEmitMeta = false; + + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode) { + // We always need a string table for this type of serializer. + StrTab.emplace(); + } + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + StringTable StrTab) + : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode, std::move(StrTab)) {} + + /// Override to emit the metadata if necessary. + void emit(const Remark &Remark) override; + + std::unique_ptr<MetaSerializer> + metaSerializer(raw_ostream &OS, + Optional<StringRef> ExternalFilename = None) override; + + static bool classof(const RemarkSerializer *S) { + return S->SerializerFormat == Format::YAMLStrTab; + } +}; + +struct YAMLStrTabMetaSerializer : public YAMLMetaSerializer { + /// The string table is part of the metadata. + const StringTable &StrTab; + + YAMLStrTabMetaSerializer(raw_ostream &OS, + Optional<StringRef> ExternalFilename, + const StringTable &StrTab) + : YAMLMetaSerializer(OS, ExternalFilename), StrTab(StrTab) {} + + void emit() override; +}; + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_SERIALIZER_H */ diff --git a/include/llvm/Support/AArch64TargetParser.def b/include/llvm/Support/AArch64TargetParser.def index e152f383b3ec..15737265dfc3 100644 --- a/include/llvm/Support/AArch64TargetParser.def +++ b/include/llvm/Support/AArch64TargetParser.def @@ -50,35 +50,36 @@ AARCH64_ARCH("armv8.5-a", ARMV8_5A, "8.5-A", "v8.5a", #define AARCH64_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) #endif // FIXME: This would be nicer were it tablegen -AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") -AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") -AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") -AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") -AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") -AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") -AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") -AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") -AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") -AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") -AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") -AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") -AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") -AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") -AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") -AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") -AARCH64_ARCH_EXT_NAME("sve2", AArch64::AEK_SVE2, "+sve2", "-sve2") -AARCH64_ARCH_EXT_NAME("sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes") -AARCH64_ARCH_EXT_NAME("sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4") -AARCH64_ARCH_EXT_NAME("sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3") -AARCH64_ARCH_EXT_NAME("bitperm", AArch64::AEK_BITPERM, "+bitperm", "-bitperm") -AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") -AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") -AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") -AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") -AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") -AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") +AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") +AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") +AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") +AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") +AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") +AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") +AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") +AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") +AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") +AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") +AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") +AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") +AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") +AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") +AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") +AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") +AARCH64_ARCH_EXT_NAME("sve2", AArch64::AEK_SVE2, "+sve2", "-sve2") +AARCH64_ARCH_EXT_NAME("sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes") +AARCH64_ARCH_EXT_NAME("sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4") +AARCH64_ARCH_EXT_NAME("sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3") +AARCH64_ARCH_EXT_NAME("sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm") +AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") +AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") +AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") +AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") +AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") +AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") +AARCH64_ARCH_EXT_NAME("tme", AArch64::AEK_TME, "+tme", "-tme") #undef AARCH64_ARCH_EXT_NAME #ifndef AARCH64_CPU_NAME @@ -92,6 +93,12 @@ AARCH64_CPU_NAME("cortex-a55", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC)) AARCH64_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) +AARCH64_CPU_NAME("cortex-a65", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + AArch64::AEK_RCPC | AArch64::AEK_SSBS)) +AARCH64_CPU_NAME("cortex-a65ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + AArch64::AEK_RCPC | AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, @@ -104,6 +111,13 @@ AARCH64_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, AARCH64_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | AArch64::AEK_SSBS)) +AARCH64_CPU_NAME("neoverse-e1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + AArch64::AEK_RCPC | AArch64::AEK_SSBS)) +AARCH64_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | + AArch64::AEK_PROFILE | AArch64::AEK_RAS | AArch64::AEK_RCPC | + AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) AARCH64_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, diff --git a/include/llvm/Support/AArch64TargetParser.h b/include/llvm/Support/AArch64TargetParser.h index 965d38535e74..94f341c83260 100644 --- a/include/llvm/Support/AArch64TargetParser.h +++ b/include/llvm/Support/AArch64TargetParser.h @@ -53,7 +53,8 @@ enum ArchExtKind : unsigned { AEK_SVE2AES = 1 << 24, AEK_SVE2SM4 = 1 << 25, AEK_SVE2SHA3 = 1 << 26, - AEK_BITPERM = 1 << 27, + AEK_SVE2BITPERM = 1 << 27, + AEK_TME = 1 << 28, }; enum class ArchKind { diff --git a/include/llvm/Support/ARMTargetParser.def b/include/llvm/Support/ARMTargetParser.def index f466b3252748..3e77e20762c1 100644 --- a/include/llvm/Support/ARMTargetParser.def +++ b/include/llvm/Support/ARMTargetParser.def @@ -274,6 +274,8 @@ ARM_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) diff --git a/include/llvm/Support/ARMTargetParser.h b/include/llvm/Support/ARMTargetParser.h index 4b9070dea596..02d4c975129f 100644 --- a/include/llvm/Support/ARMTargetParser.h +++ b/include/llvm/Support/ARMTargetParser.h @@ -39,19 +39,13 @@ enum ArchExtKind : unsigned { AEK_DSP = 1 << 10, AEK_FP16 = 1 << 11, AEK_RAS = 1 << 12, - AEK_SVE = 1 << 13, - AEK_DOTPROD = 1 << 14, - AEK_SHA2 = 1 << 15, - AEK_AES = 1 << 16, - AEK_FP16FML = 1 << 17, - AEK_SB = 1 << 18, - AEK_SVE2 = 1 << 19, - AEK_SVE2AES = 1 << 20, - AEK_SVE2SM4 = 1 << 21, - AEK_SVE2SHA3 = 1 << 22, - AEK_BITPERM = 1 << 23, - AEK_FP_DP = 1 << 24, - AEK_LOB = 1 << 25, + AEK_DOTPROD = 1 << 13, + AEK_SHA2 = 1 << 14, + AEK_AES = 1 << 15, + AEK_FP16FML = 1 << 16, + AEK_SB = 1 << 17, + AEK_FP_DP = 1 << 18, + AEK_LOB = 1 << 19, // Unsupported extensions. AEK_OS = 0x8000000, AEK_IWMMXT = 0x10000000, diff --git a/include/llvm/Support/AlignOf.h b/include/llvm/Support/AlignOf.h index d12401f0eb49..eb42542b777f 100644 --- a/include/llvm/Support/AlignOf.h +++ b/include/llvm/Support/AlignOf.h @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file defines the AlignedCharArray and AlignedCharArrayUnion classes. +// This file defines the AlignedCharArrayUnion class. // //===----------------------------------------------------------------------===// @@ -18,128 +18,38 @@ namespace llvm { -/// \struct AlignedCharArray -/// Helper for building an aligned character array type. -/// -/// This template is used to explicitly build up a collection of aligned -/// character array types. We have to build these up using a macro and explicit -/// specialization to cope with MSVC (at least till 2015) where only an -/// integer literal can be used to specify an alignment constraint. Once built -/// up here, we can then begin to indirect between these using normal C++ -/// template parameters. - -// MSVC requires special handling here. -#ifndef _MSC_VER - -template<std::size_t Alignment, std::size_t Size> -struct AlignedCharArray { - alignas(Alignment) char buffer[Size]; -}; - -#else // _MSC_VER - -/// Create a type with an aligned char buffer. -template<std::size_t Alignment, std::size_t Size> -struct AlignedCharArray; - -// We provide special variations of this template for the most common -// alignments because __declspec(align(...)) doesn't actually work when it is -// a member of a by-value function argument in MSVC, even if the alignment -// request is something reasonably like 8-byte or 16-byte. Note that we can't -// even include the declspec with the union that forces the alignment because -// MSVC warns on the existence of the declspec despite the union member forcing -// proper alignment. - -template<std::size_t Size> -struct AlignedCharArray<1, Size> { - union { - char aligned; - char buffer[Size]; - }; -}; - -template<std::size_t Size> -struct AlignedCharArray<2, Size> { - union { - short aligned; - char buffer[Size]; - }; -}; - -template<std::size_t Size> -struct AlignedCharArray<4, Size> { - union { - int aligned; - char buffer[Size]; - }; -}; +namespace detail { -template<std::size_t Size> -struct AlignedCharArray<8, Size> { - union { - double aligned; - char buffer[Size]; - }; +template <typename T, typename... Ts> class AlignerImpl { + T t; + AlignerImpl<Ts...> rest; + AlignerImpl() = delete; }; - -// The rest of these are provided with a __declspec(align(...)) and we simply -// can't pass them by-value as function arguments on MSVC. - -#define LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(x) \ - template<std::size_t Size> \ - struct AlignedCharArray<x, Size> { \ - __declspec(align(x)) char buffer[Size]; \ - }; - -LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(16) -LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(32) -LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(64) -LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(128) - -#undef LLVM_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT - -#endif // _MSC_VER - -namespace detail { -template <typename T1, - typename T2 = char, typename T3 = char, typename T4 = char, - typename T5 = char, typename T6 = char, typename T7 = char, - typename T8 = char, typename T9 = char, typename T10 = char> -class AlignerImpl { - T1 t1; T2 t2; T3 t3; T4 t4; T5 t5; T6 t6; T7 t7; T8 t8; T9 t9; T10 t10; - +template <typename T> class AlignerImpl<T> { + T t; AlignerImpl() = delete; }; -template <typename T1, - typename T2 = char, typename T3 = char, typename T4 = char, - typename T5 = char, typename T6 = char, typename T7 = char, - typename T8 = char, typename T9 = char, typename T10 = char> -union SizerImpl { - char arr1[sizeof(T1)], arr2[sizeof(T2)], arr3[sizeof(T3)], arr4[sizeof(T4)], - arr5[sizeof(T5)], arr6[sizeof(T6)], arr7[sizeof(T7)], arr8[sizeof(T8)], - arr9[sizeof(T9)], arr10[sizeof(T10)]; +template <typename T, typename... Ts> union SizerImpl { + char arr[sizeof(T)]; + SizerImpl<Ts...> rest; }; + +template <typename T> union SizerImpl<T> { char arr[sizeof(T)]; }; } // end namespace detail -/// This union template exposes a suitably aligned and sized character -/// array member which can hold elements of any of up to ten types. +/// A suitably aligned and sized character array member which can hold elements +/// of any type. /// -/// These types may be arrays, structs, or any other types. The goal is to -/// expose a char array buffer member which can be used as suitable storage for -/// a placement new of any of these types. Support for more than ten types can -/// be added at the cost of more boilerplate. -template <typename T1, - typename T2 = char, typename T3 = char, typename T4 = char, - typename T5 = char, typename T6 = char, typename T7 = char, - typename T8 = char, typename T9 = char, typename T10 = char> -struct AlignedCharArrayUnion : llvm::AlignedCharArray< - alignof(llvm::detail::AlignerImpl<T1, T2, T3, T4, T5, - T6, T7, T8, T9, T10>), - sizeof(::llvm::detail::SizerImpl<T1, T2, T3, T4, T5, - T6, T7, T8, T9, T10>)> { +/// These types may be arrays, structs, or any other types. This exposes a +/// `buffer` member which can be used as suitable storage for a placement new of +/// any of these types. +template <typename T, typename... Ts> struct AlignedCharArrayUnion { + alignas(::llvm::detail::AlignerImpl<T, Ts...>) char buffer[sizeof( + llvm::detail::SizerImpl<T, Ts...>)]; }; + } // end namespace llvm #endif // LLVM_SUPPORT_ALIGNOF_H diff --git a/include/llvm/Support/Alignment.h b/include/llvm/Support/Alignment.h new file mode 100644 index 000000000000..72fad87dd0d4 --- /dev/null +++ b/include/llvm/Support/Alignment.h @@ -0,0 +1,403 @@ +//===-- llvm/Support/Alignment.h - Useful alignment functions ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains types to represent alignments. +// They are instrumented to guarantee some invariants are preserved and prevent +// invalid manipulations. +// +// - Align represents an alignment in bytes, it is always set and always a valid +// power of two, its minimum value is 1 which means no alignment requirements. +// +// - MaybeAlign is an optional type, it may be undefined or set. When it's set +// you can get the underlying Align type by using the getValue() method. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_ALIGNMENT_H_ +#define LLVM_SUPPORT_ALIGNMENT_H_ + +#include "llvm/ADT/Optional.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/MathExtras.h" +#include <cassert> +#include <limits> + +namespace llvm { + +#define ALIGN_CHECK_ISPOSITIVE(decl) \ + assert(decl > 0 && (#decl " should be defined")) +#define ALIGN_CHECK_ISSET(decl) \ + assert(decl.hasValue() && (#decl " should be defined")) + +/// This struct is a compact representation of a valid (non-zero power of two) +/// alignment. +/// It is suitable for use as static global constants. +struct Align { +private: + uint8_t ShiftValue = 0; /// The log2 of the required alignment. + /// ShiftValue is less than 64 by construction. + + friend struct MaybeAlign; + friend unsigned Log2(Align); + friend bool operator==(Align Lhs, Align Rhs); + friend bool operator!=(Align Lhs, Align Rhs); + friend bool operator<=(Align Lhs, Align Rhs); + friend bool operator>=(Align Lhs, Align Rhs); + friend bool operator<(Align Lhs, Align Rhs); + friend bool operator>(Align Lhs, Align Rhs); + friend unsigned encode(struct MaybeAlign A); + friend struct MaybeAlign decodeMaybeAlign(unsigned Value); + + /// A trivial type to allow construction of constexpr Align. + /// This is currently needed to workaround a bug in GCC 5.3 which prevents + /// definition of constexpr assign operators. + /// https://stackoverflow.com/questions/46756288/explicitly-defaulted-function-cannot-be-declared-as-constexpr-because-the-implic + /// FIXME: Remove this, make all assign operators constexpr and introduce user + /// defined literals when we don't have to support GCC 5.3 anymore. + /// https://llvm.org/docs/GettingStarted.html#getting-a-modern-host-c-toolchain + struct LogValue { + uint8_t Log; + }; + +public: + /// Default is byte-aligned. + constexpr Align() = default; + /// Do not perform checks in case of copy/move construct/assign, because the + /// checks have been performed when building `Other`. + constexpr Align(const Align &Other) = default; + constexpr Align(Align &&Other) = default; + Align &operator=(const Align &Other) = default; + Align &operator=(Align &&Other) = default; + + explicit Align(uint64_t Value) { + assert(Value > 0 && "Value must not be 0"); + assert(llvm::isPowerOf2_64(Value) && "Alignment is not a power of 2"); + ShiftValue = Log2_64(Value); + assert(ShiftValue < 64 && "Broken invariant"); + } + + /// This is a hole in the type system and should not be abused. + /// Needed to interact with C for instance. + uint64_t value() const { return uint64_t(1) << ShiftValue; } + + /// Returns a default constructed Align which corresponds to no alignment. + /// This is useful to test for unalignment as it conveys clear semantic. + /// `if (A != Align::None())` + /// would be better than + /// `if (A > Align(1))` + constexpr static const Align None() { return Align(); } + + /// Allow constructions of constexpr Align. + template <size_t kValue> constexpr static LogValue Constant() { + return LogValue{static_cast<uint8_t>(CTLog2<kValue>())}; + } + + /// Allow constructions of constexpr Align from types. + /// Compile time equivalent to Align(alignof(T)). + template <typename T> constexpr static LogValue Of() { + return Constant<std::alignment_of<T>::value>(); + } + + /// Constexpr constructor from LogValue type. + constexpr Align(LogValue CA) : ShiftValue(CA.Log) {} +}; + +/// Treats the value 0 as a 1, so Align is always at least 1. +inline Align assumeAligned(uint64_t Value) { + return Value ? Align(Value) : Align(); +} + +/// This struct is a compact representation of a valid (power of two) or +/// undefined (0) alignment. +struct MaybeAlign : public llvm::Optional<Align> { +private: + using UP = llvm::Optional<Align>; + +public: + /// Default is undefined. + MaybeAlign() = default; + /// Do not perform checks in case of copy/move construct/assign, because the + /// checks have been performed when building `Other`. + MaybeAlign(const MaybeAlign &Other) = default; + MaybeAlign &operator=(const MaybeAlign &Other) = default; + MaybeAlign(MaybeAlign &&Other) = default; + MaybeAlign &operator=(MaybeAlign &&Other) = default; + + /// Use llvm::Optional<Align> constructor. + using UP::UP; + + explicit MaybeAlign(uint64_t Value) { + assert((Value == 0 || llvm::isPowerOf2_64(Value)) && + "Alignment is neither 0 nor a power of 2"); + if (Value) + emplace(Value); + } + + /// For convenience, returns a valid alignment or 1 if undefined. + Align valueOrOne() const { return hasValue() ? getValue() : Align(); } +}; + +/// Checks that SizeInBytes is a multiple of the alignment. +inline bool isAligned(Align Lhs, uint64_t SizeInBytes) { + return SizeInBytes % Lhs.value() == 0; +} + +/// Checks that SizeInBytes is a multiple of the alignment. +/// Returns false if the alignment is undefined. +inline bool isAligned(MaybeAlign Lhs, uint64_t SizeInBytes) { + ALIGN_CHECK_ISSET(Lhs); + return SizeInBytes % (*Lhs).value() == 0; +} + +/// Checks that Addr is a multiple of the alignment. +inline bool isAddrAligned(Align Lhs, const void *Addr) { + return isAligned(Lhs, reinterpret_cast<uintptr_t>(Addr)); +} + +/// Returns a multiple of A needed to store `Size` bytes. +inline uint64_t alignTo(uint64_t Size, Align A) { + const uint64_t value = A.value(); + // The following line is equivalent to `(Size + value - 1) / value * value`. + + // The division followed by a multiplication can be thought of as a right + // shift followed by a left shift which zeros out the extra bits produced in + // the bump; `~(value - 1)` is a mask where all those bits being zeroed out + // are just zero. + + // Most compilers can generate this code but the pattern may be missed when + // multiple functions gets inlined. + return (Size + value - 1) & ~(value - 1); +} + +/// Returns a multiple of A needed to store `Size` bytes. +/// Returns `Size` if current alignment is undefined. +inline uint64_t alignTo(uint64_t Size, MaybeAlign A) { + return A ? alignTo(Size, A.getValue()) : Size; +} + +/// Aligns `Addr` to `Alignment` bytes, rounding up. +inline uintptr_t alignAddr(const void *Addr, Align Alignment) { + uintptr_t ArithAddr = reinterpret_cast<uintptr_t>(Addr); + assert(static_cast<uintptr_t>(ArithAddr + Alignment.value() - 1) >= + ArithAddr && "Overflow"); + return alignTo(ArithAddr, Alignment); +} + +/// Returns the offset to the next integer (mod 2**64) that is greater than +/// or equal to \p Value and is a multiple of \p Align. +inline uint64_t offsetToAlignment(uint64_t Value, Align Alignment) { + return alignTo(Value, Alignment) - Value; +} + +/// Returns the necessary adjustment for aligning `Addr` to `Alignment` +/// bytes, rounding up. +inline uint64_t offsetToAlignedAddr(const void *Addr, Align Alignment) { + return offsetToAlignment(reinterpret_cast<uintptr_t>(Addr), Alignment); +} + +/// Returns the log2 of the alignment. +inline unsigned Log2(Align A) { return A.ShiftValue; } + +/// Returns the log2 of the alignment. +/// \pre A must be defined. +inline unsigned Log2(MaybeAlign A) { + ALIGN_CHECK_ISSET(A); + return Log2(A.getValue()); +} + +/// Returns the alignment that satisfies both alignments. +/// Same semantic as MinAlign. +inline Align commonAlignment(Align A, Align B) { return std::min(A, B); } + +/// Returns the alignment that satisfies both alignments. +/// Same semantic as MinAlign. +inline Align commonAlignment(Align A, uint64_t Offset) { + return Align(MinAlign(A.value(), Offset)); +} + +/// Returns the alignment that satisfies both alignments. +/// Same semantic as MinAlign. +inline MaybeAlign commonAlignment(MaybeAlign A, MaybeAlign B) { + return A && B ? commonAlignment(*A, *B) : A ? A : B; +} + +/// Returns the alignment that satisfies both alignments. +/// Same semantic as MinAlign. +inline MaybeAlign commonAlignment(MaybeAlign A, uint64_t Offset) { + return MaybeAlign(MinAlign((*A).value(), Offset)); +} + +/// Returns a representation of the alignment that encodes undefined as 0. +inline unsigned encode(MaybeAlign A) { return A ? A->ShiftValue + 1 : 0; } + +/// Dual operation of the encode function above. +inline MaybeAlign decodeMaybeAlign(unsigned Value) { + if (Value == 0) + return MaybeAlign(); + Align Out; + Out.ShiftValue = Value - 1; + return Out; +} + +/// Returns a representation of the alignment, the encoded value is positive by +/// definition. +inline unsigned encode(Align A) { return encode(MaybeAlign(A)); } + +/// Comparisons between Align and scalars. Rhs must be positive. +inline bool operator==(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() == Rhs; +} +inline bool operator!=(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() != Rhs; +} +inline bool operator<=(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() <= Rhs; +} +inline bool operator>=(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() >= Rhs; +} +inline bool operator<(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() < Rhs; +} +inline bool operator>(Align Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISPOSITIVE(Rhs); + return Lhs.value() > Rhs; +} + +/// Comparisons between MaybeAlign and scalars. +inline bool operator==(MaybeAlign Lhs, uint64_t Rhs) { + return Lhs ? (*Lhs).value() == Rhs : Rhs == 0; +} +inline bool operator!=(MaybeAlign Lhs, uint64_t Rhs) { + return Lhs ? (*Lhs).value() != Rhs : Rhs != 0; +} +inline bool operator<=(MaybeAlign Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISSET(Lhs); + ALIGN_CHECK_ISPOSITIVE(Rhs); + return (*Lhs).value() <= Rhs; +} +inline bool operator>=(MaybeAlign Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISSET(Lhs); + ALIGN_CHECK_ISPOSITIVE(Rhs); + return (*Lhs).value() >= Rhs; +} +inline bool operator<(MaybeAlign Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISSET(Lhs); + ALIGN_CHECK_ISPOSITIVE(Rhs); + return (*Lhs).value() < Rhs; +} +inline bool operator>(MaybeAlign Lhs, uint64_t Rhs) { + ALIGN_CHECK_ISSET(Lhs); + ALIGN_CHECK_ISPOSITIVE(Rhs); + return (*Lhs).value() > Rhs; +} + +/// Comparisons operators between Align. +inline bool operator==(Align Lhs, Align Rhs) { + return Lhs.ShiftValue == Rhs.ShiftValue; +} +inline bool operator!=(Align Lhs, Align Rhs) { + return Lhs.ShiftValue != Rhs.ShiftValue; +} +inline bool operator<=(Align Lhs, Align Rhs) { + return Lhs.ShiftValue <= Rhs.ShiftValue; +} +inline bool operator>=(Align Lhs, Align Rhs) { + return Lhs.ShiftValue >= Rhs.ShiftValue; +} +inline bool operator<(Align Lhs, Align Rhs) { + return Lhs.ShiftValue < Rhs.ShiftValue; +} +inline bool operator>(Align Lhs, Align Rhs) { + return Lhs.ShiftValue > Rhs.ShiftValue; +} + +/// Comparisons operators between Align and MaybeAlign. +inline bool operator==(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() == (*Rhs).value(); +} +inline bool operator!=(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() != (*Rhs).value(); +} +inline bool operator<=(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() <= (*Rhs).value(); +} +inline bool operator>=(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() >= (*Rhs).value(); +} +inline bool operator<(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() < (*Rhs).value(); +} +inline bool operator>(Align Lhs, MaybeAlign Rhs) { + ALIGN_CHECK_ISSET(Rhs); + return Lhs.value() > (*Rhs).value(); +} + +/// Comparisons operators between MaybeAlign and Align. +inline bool operator==(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() == Rhs.value(); +} +inline bool operator!=(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() != Rhs.value(); +} +inline bool operator<=(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() <= Rhs.value(); +} +inline bool operator>=(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() >= Rhs.value(); +} +inline bool operator<(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() < Rhs.value(); +} +inline bool operator>(MaybeAlign Lhs, Align Rhs) { + ALIGN_CHECK_ISSET(Lhs); + return Lhs && (*Lhs).value() > Rhs.value(); +} + +inline Align operator/(Align Lhs, uint64_t Divisor) { + assert(llvm::isPowerOf2_64(Divisor) && + "Divisor must be positive and a power of 2"); + assert(Lhs != 1 && "Can't halve byte alignment"); + return Align(Lhs.value() / Divisor); +} + +inline MaybeAlign operator/(MaybeAlign Lhs, uint64_t Divisor) { + assert(llvm::isPowerOf2_64(Divisor) && + "Divisor must be positive and a power of 2"); + return Lhs ? Lhs.getValue() / Divisor : MaybeAlign(); +} + +inline Align max(MaybeAlign Lhs, Align Rhs) { + return Lhs && *Lhs > Rhs ? *Lhs : Rhs; +} + +inline Align max(Align Lhs, MaybeAlign Rhs) { + return Rhs && *Rhs > Lhs ? *Rhs : Lhs; +} + +#undef ALIGN_CHECK_ISPOSITIVE +#undef ALIGN_CHECK_ISSET + +} // namespace llvm + +#endif // LLVM_SUPPORT_ALIGNMENT_H_ diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h index 09e967b98abc..106b90c35bf5 100644 --- a/include/llvm/Support/Allocator.h +++ b/include/llvm/Support/Allocator.h @@ -22,6 +22,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -211,13 +212,11 @@ public: /// Allocate space at the specified alignment. LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * - Allocate(size_t Size, size_t Alignment) { - assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); - + Allocate(size_t Size, Align Alignment) { // Keep track of how many bytes we've allocated. BytesAllocated += Size; - size_t Adjustment = alignmentAdjustment(CurPtr, Alignment); + size_t Adjustment = offsetToAlignedAddr(CurPtr, Alignment); assert(Adjustment + Size >= Size && "Adjustment + Size must not overflow"); size_t SizeToAllocate = Size; @@ -240,7 +239,7 @@ public: } // If Size is really big, allocate a separate slab for it. - size_t PaddedSize = SizeToAllocate + Alignment - 1; + size_t PaddedSize = SizeToAllocate + Alignment.value() - 1; if (PaddedSize > SizeThreshold) { void *NewSlab = Allocator.Allocate(PaddedSize, 0); // We own the new slab and don't want anyone reading anyting other than @@ -268,6 +267,12 @@ public: return AlignedPtr; } + inline LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * + Allocate(size_t Size, size_t Alignment) { + assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); + return Allocate(Size, Align(Alignment)); + } + // Pull in base class overloads. using AllocatorBase<BumpPtrAllocatorImpl>::Allocate; @@ -461,7 +466,7 @@ public: /// all memory allocated so far. void DestroyAll() { auto DestroyElements = [](char *Begin, char *End) { - assert(Begin == (char *)alignAddr(Begin, alignof(T))); + assert(Begin == (char *)alignAddr(Begin, Align::Of<T>())); for (char *Ptr = Begin; Ptr + sizeof(T) <= End; Ptr += sizeof(T)) reinterpret_cast<T *>(Ptr)->~T(); }; @@ -470,7 +475,7 @@ public: ++I) { size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize( std::distance(Allocator.Slabs.begin(), I)); - char *Begin = (char *)alignAddr(*I, alignof(T)); + char *Begin = (char *)alignAddr(*I, Align::Of<T>()); char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr : (char *)*I + AllocatedSlabSize; @@ -480,7 +485,8 @@ public: for (auto &PtrAndSize : Allocator.CustomSizedSlabs) { void *Ptr = PtrAndSize.first; size_t Size = PtrAndSize.second; - DestroyElements((char *)alignAddr(Ptr, alignof(T)), (char *)Ptr + Size); + DestroyElements((char *)alignAddr(Ptr, Align::Of<T>()), + (char *)Ptr + Size); } Allocator.Reset(); diff --git a/include/llvm/Support/Automaton.h b/include/llvm/Support/Automaton.h new file mode 100644 index 000000000000..7c13a698e492 --- /dev/null +++ b/include/llvm/Support/Automaton.h @@ -0,0 +1,253 @@ +//===-- Automaton.h - Support for driving TableGen-produced DFAs ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements class that drive and introspect deterministic finite- +// state automata (DFAs) as generated by TableGen's -gen-automata backend. +// +// For a description of how to define an automaton, see +// include/llvm/TableGen/Automaton.td. +// +// One important detail is that these deterministic automata are created from +// (potentially) nondeterministic definitions. Therefore a unique sequence of +// input symbols will produce one path through the DFA but multiple paths +// through the original NFA. An automaton by default only returns "accepted" or +// "not accepted", but frequently we want to analyze what NFA path was taken. +// Finding a path through the NFA states that results in a DFA state can help +// answer *what* the solution to a problem was, not just that there exists a +// solution. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_AUTOMATON_H +#define LLVM_SUPPORT_AUTOMATON_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Allocator.h" +#include <deque> +#include <map> +#include <memory> +#include <unordered_map> +#include <vector> + +namespace llvm { + +using NfaPath = SmallVector<uint64_t, 4>; + +/// Forward define the pair type used by the automata transition info tables. +/// +/// Experimental results with large tables have shown a significant (multiple +/// orders of magnitude) parsing speedup by using a custom struct here with a +/// trivial constructor rather than std::pair<uint64_t, uint64_t>. +struct NfaStatePair { + uint64_t FromDfaState, ToDfaState; + + bool operator<(const NfaStatePair &Other) const { + return std::make_tuple(FromDfaState, ToDfaState) < + std::make_tuple(Other.FromDfaState, Other.ToDfaState); + } +}; + +namespace internal { +/// The internal class that maintains all possible paths through an NFA based +/// on a path through the DFA. +class NfaTranscriber { +private: + /// Cached transition table. This is a table of NfaStatePairs that contains + /// zero-terminated sequences pointed to by DFA transitions. + ArrayRef<NfaStatePair> TransitionInfo; + + /// A simple linked-list of traversed states that can have a shared tail. The + /// traversed path is stored in reverse order with the latest state as the + /// head. + struct PathSegment { + uint64_t State; + PathSegment *Tail; + }; + + /// We allocate segment objects frequently. Allocate them upfront and dispose + /// at the end of a traversal rather than hammering the system allocator. + SpecificBumpPtrAllocator<PathSegment> Allocator; + + /// Heads of each tracked path. These are not ordered. + std::deque<PathSegment *> Heads; + + /// The returned paths. This is populated during getPaths. + SmallVector<NfaPath, 4> Paths; + + /// Create a new segment and return it. + PathSegment *makePathSegment(uint64_t State, PathSegment *Tail) { + PathSegment *P = Allocator.Allocate(); + *P = {State, Tail}; + return P; + } + + /// Pairs defines a sequence of possible NFA transitions for a single DFA + /// transition. + void transition(ArrayRef<NfaStatePair> Pairs) { + // Iterate over all existing heads. We will mutate the Heads deque during + // iteration. + unsigned NumHeads = Heads.size(); + for (unsigned I = 0; I < NumHeads; ++I) { + PathSegment *Head = Heads[I]; + // The sequence of pairs is sorted. Select the set of pairs that + // transition from the current head state. + auto PI = lower_bound(Pairs, NfaStatePair{Head->State, 0ULL}); + auto PE = upper_bound(Pairs, NfaStatePair{Head->State, INT64_MAX}); + // For every transition from the current head state, add a new path + // segment. + for (; PI != PE; ++PI) + if (PI->FromDfaState == Head->State) + Heads.push_back(makePathSegment(PI->ToDfaState, Head)); + } + // Now we've iterated over all the initial heads and added new ones, + // dispose of the original heads. + Heads.erase(Heads.begin(), std::next(Heads.begin(), NumHeads)); + } + +public: + NfaTranscriber(ArrayRef<NfaStatePair> TransitionInfo) + : TransitionInfo(TransitionInfo) { + reset(); + } + + void reset() { + Paths.clear(); + Heads.clear(); + Allocator.DestroyAll(); + // The initial NFA state is 0. + Heads.push_back(makePathSegment(0ULL, nullptr)); + } + + void transition(unsigned TransitionInfoIdx) { + unsigned EndIdx = TransitionInfoIdx; + while (TransitionInfo[EndIdx].ToDfaState != 0) + ++EndIdx; + ArrayRef<NfaStatePair> Pairs(&TransitionInfo[TransitionInfoIdx], + EndIdx - TransitionInfoIdx); + transition(Pairs); + } + + ArrayRef<NfaPath> getPaths() { + Paths.clear(); + for (auto *Head : Heads) { + NfaPath P; + while (Head->State != 0) { + P.push_back(Head->State); + Head = Head->Tail; + } + std::reverse(P.begin(), P.end()); + Paths.push_back(std::move(P)); + } + return Paths; + } +}; +} // namespace internal + +/// A deterministic finite-state automaton. The automaton is defined in +/// TableGen; this object drives an automaton defined by tblgen-emitted tables. +/// +/// An automaton accepts a sequence of input tokens ("actions"). This class is +/// templated on the type of these actions. +template <typename ActionT> class Automaton { + /// Map from {State, Action} to {NewState, TransitionInfoIdx}. + /// TransitionInfoIdx is used by the DfaTranscriber to analyze the transition. + /// FIXME: This uses a std::map because ActionT can be a pair type including + /// an enum. In particular DenseMapInfo<ActionT> must be defined to use + /// DenseMap here. + /// This is a shared_ptr to allow very quick copy-construction of Automata; this + /// state is immutable after construction so this is safe. + using MapTy = std::map<std::pair<uint64_t, ActionT>, std::pair<uint64_t, unsigned>>; + std::shared_ptr<MapTy> M; + /// An optional transcription object. This uses much more state than simply + /// traversing the DFA for acceptance, so is heap allocated. + std::shared_ptr<internal::NfaTranscriber> Transcriber; + /// The initial DFA state is 1. + uint64_t State = 1; + /// True if we should transcribe and false if not (even if Transcriber is defined). + bool Transcribe; + +public: + /// Create an automaton. + /// \param Transitions The Transitions table as created by TableGen. Note that + /// because the action type differs per automaton, the + /// table type is templated as ArrayRef<InfoT>. + /// \param TranscriptionTable The TransitionInfo table as created by TableGen. + /// + /// Providing the TranscriptionTable argument as non-empty will enable the + /// use of transcription, which analyzes the possible paths in the original + /// NFA taken by the DFA. NOTE: This is substantially more work than simply + /// driving the DFA, so unless you require the getPaths() method leave this + /// empty. + template <typename InfoT> + Automaton(ArrayRef<InfoT> Transitions, + ArrayRef<NfaStatePair> TranscriptionTable = {}) { + if (!TranscriptionTable.empty()) + Transcriber = + std::make_shared<internal::NfaTranscriber>(TranscriptionTable); + Transcribe = Transcriber != nullptr; + M = std::make_shared<MapTy>(); + for (const auto &I : Transitions) + // Greedily read and cache the transition table. + M->emplace(std::make_pair(I.FromDfaState, I.Action), + std::make_pair(I.ToDfaState, I.InfoIdx)); + } + Automaton(const Automaton &) = default; + + /// Reset the automaton to its initial state. + void reset() { + State = 1; + if (Transcriber) + Transcriber->reset(); + } + + /// Enable or disable transcription. Transcription is only available if + /// TranscriptionTable was provided to the constructor. + void enableTranscription(bool Enable = true) { + assert(Transcriber && + "Transcription is only available if TranscriptionTable was provided " + "to the Automaton constructor"); + Transcribe = Enable; + } + + /// Transition the automaton based on input symbol A. Return true if the + /// automaton transitioned to a valid state, false if the automaton + /// transitioned to an invalid state. + /// + /// If this function returns false, all methods are undefined until reset() is + /// called. + bool add(const ActionT &A) { + auto I = M->find({State, A}); + if (I == M->end()) + return false; + if (Transcriber && Transcribe) + Transcriber->transition(I->second.second); + State = I->second.first; + return true; + } + + /// Return true if the automaton can be transitioned based on input symbol A. + bool canAdd(const ActionT &A) { + auto I = M->find({State, A}); + return I != M->end(); + } + + /// Obtain a set of possible paths through the input nondeterministic + /// automaton that could be obtained from the sequence of input actions + /// presented to this deterministic automaton. + ArrayRef<NfaPath> getNfaPaths() { + assert(Transcriber && Transcribe && + "Can only obtain NFA paths if transcribing!"); + return Transcriber->getPaths(); + } +}; + +} // namespace llvm + +#endif // LLVM_SUPPORT_AUTOMATON_H diff --git a/include/llvm/Support/BinaryStreamArray.h b/include/llvm/Support/BinaryStreamArray.h index 96d09db69ae5..67ba2e4189be 100644 --- a/include/llvm/Support/BinaryStreamArray.h +++ b/include/llvm/Support/BinaryStreamArray.h @@ -286,7 +286,7 @@ public: // an exact multiple of the element size. consumeError(std::move(EC)); } - assert(llvm::alignmentAdjustment(Data.data(), alignof(T)) == 0); + assert(isAddrAligned(Align::Of<T>(), Data.data())); return *reinterpret_cast<const T *>(Data.data()); } diff --git a/include/llvm/Support/BinaryStreamReader.h b/include/llvm/Support/BinaryStreamReader.h index d8fddde66bfa..9e16ce227ff8 100644 --- a/include/llvm/Support/BinaryStreamReader.h +++ b/include/llvm/Support/BinaryStreamReader.h @@ -198,7 +198,7 @@ public: if (auto EC = readBytes(Bytes, NumElements * sizeof(T))) return EC; - assert(alignmentAdjustment(Bytes.data(), alignof(T)) == 0 && + assert(isAddrAligned(Align::Of<T>(), Bytes.data()) && "Reading at invalid alignment!"); Array = ArrayRef<T>(reinterpret_cast<const T *>(Bytes.data()), NumElements); diff --git a/include/llvm/Support/CRC.h b/include/llvm/Support/CRC.h index 6ea8e3edcea4..210890ae06d4 100644 --- a/include/llvm/Support/CRC.h +++ b/include/llvm/Support/CRC.h @@ -6,20 +6,55 @@ // //===----------------------------------------------------------------------===// // -// This file contains basic functions for calculating Cyclic Redundancy Check -// or CRC. +// This file contains implementations of CRC functions. // //===----------------------------------------------------------------------===// #ifndef LLVM_SUPPORT_CRC_H #define LLVM_SUPPORT_CRC_H -#include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" namespace llvm { -/// zlib independent CRC32 calculation. -uint32_t crc32(uint32_t CRC, StringRef S); +template <typename T> class ArrayRef; + +// Compute the CRC-32 of Data. +uint32_t crc32(ArrayRef<uint8_t> Data); + +// Compute the running CRC-32 of Data, with CRC being the previous value of the +// checksum. +uint32_t crc32(uint32_t CRC, ArrayRef<uint8_t> Data); + +// Class for computing the JamCRC. +// +// We will use the "Rocksoft^tm Model CRC Algorithm" to describe the properties +// of this CRC: +// Width : 32 +// Poly : 04C11DB7 +// Init : FFFFFFFF +// RefIn : True +// RefOut : True +// XorOut : 00000000 +// Check : 340BC6D9 (result of CRC for "123456789") +// +// In other words, this is the same as CRC-32, except that XorOut is 0 instead +// of FFFFFFFF. +// +// N.B. We permit flexibility of the "Init" value. Some consumers of this need +// it to be zero. +class JamCRC { +public: + JamCRC(uint32_t Init = 0xFFFFFFFFU) : CRC(Init) {} + + // Update the CRC calculation with Data. + void update(ArrayRef<uint8_t> Data); + + uint32_t getCRC() const { return CRC; } + +private: + uint32_t CRC; +}; + } // end namespace llvm #endif diff --git a/include/llvm/Support/CommandLine.h b/include/llvm/Support/CommandLine.h index 3cc2c3c0121b..63784463e171 100644 --- a/include/llvm/Support/CommandLine.h +++ b/include/llvm/Support/CommandLine.h @@ -2000,6 +2000,9 @@ void ResetAllOptionOccurrences(); /// where no options are supported. void ResetCommandLineParser(); +/// Parses `Arg` into the option handler `Handler`. +bool ProvidePositionalOption(Option *Handler, StringRef Arg, int i); + } // end namespace cl } // end namespace llvm diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h index 3f4f465f3960..cb7e57d4cd21 100644 --- a/include/llvm/Support/Compiler.h +++ b/include/llvm/Support/Compiler.h @@ -7,7 +7,8 @@ //===----------------------------------------------------------------------===// // // This file defines several macros, based on the current compiler. This allows -// use of compiler-specific features in a way that remains portable. +// use of compiler-specific features in a way that remains portable. This header +// can be included from either C or C++. // //===----------------------------------------------------------------------===// @@ -16,7 +17,9 @@ #include "llvm/Config/llvm-config.h" +#ifdef __cplusplus #include <new> +#endif #include <stddef.h> #if defined(_MSC_VER) @@ -35,14 +38,20 @@ # define __has_attribute(x) 0 #endif -#ifndef __has_cpp_attribute -# define __has_cpp_attribute(x) 0 -#endif - #ifndef __has_builtin # define __has_builtin(x) 0 #endif +// Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in +// C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid. +#ifndef LLVM_HAS_CPP_ATTRIBUTE +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define LLVM_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define LLVM_HAS_CPP_ATTRIBUTE(x) 0 +#endif +#endif + /// \macro LLVM_GNUC_PREREQ /// Extend the default __GNUC_PREREQ even if glibc's features.h isn't /// available. @@ -62,13 +71,21 @@ /// \macro LLVM_MSC_PREREQ /// Is the compiler MSVC of at least the specified version? /// The common \param version values to check for are: -/// * 1900: Microsoft Visual Studio 2015 / 14.0 +/// * 1910: VS2017, version 15.1 & 15.2 +/// * 1911: VS2017, version 15.3 & 15.4 +/// * 1912: VS2017, version 15.5 +/// * 1913: VS2017, version 15.6 +/// * 1914: VS2017, version 15.7 +/// * 1915: VS2017, version 15.8 +/// * 1916: VS2017, version 15.9 +/// * 1920: VS2019, version 16.0 +/// * 1921: VS2019, version 16.1 #ifdef _MSC_VER #define LLVM_MSC_PREREQ(version) (_MSC_VER >= (version)) -// We require at least MSVC 2015. -#if !LLVM_MSC_PREREQ(1900) -#error LLVM requires at least MSVC 2015. +// We require at least MSVC 2017. +#if !LLVM_MSC_PREREQ(1910) +#error LLVM requires at least MSVC 2017. #endif #else @@ -120,14 +137,18 @@ #endif /// LLVM_NODISCARD - Warn if a type or return value is discarded. -#if __cplusplus > 201402L && __has_cpp_attribute(nodiscard) + +// Use the 'nodiscard' attribute in C++17 or newer mode. +#if __cplusplus > 201402L && LLVM_HAS_CPP_ATTRIBUTE(nodiscard) #define LLVM_NODISCARD [[nodiscard]] -#elif !__cplusplus -// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious -// error when __has_cpp_attribute is given a scoped attribute in C mode. -#define LLVM_NODISCARD -#elif __has_cpp_attribute(clang::warn_unused_result) +#elif LLVM_HAS_CPP_ATTRIBUTE(clang::warn_unused_result) #define LLVM_NODISCARD [[clang::warn_unused_result]] +// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also +// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518). +// Use the 'nodiscard' attribute in C++14 mode only with GCC. +// TODO: remove this workaround when PR33518 is resolved. +#elif defined(__GNUC__) && LLVM_HAS_CPP_ATTRIBUTE(nodiscard) +#define LLVM_NODISCARD [[nodiscard]] #else #define LLVM_NODISCARD #endif @@ -139,7 +160,7 @@ // The clang-tidy check bugprone-use-after-move recognizes this attribute as a // marker that a moved-from object has left the indeterminate state and can be // reused. -#if __has_cpp_attribute(clang::reinitializes) +#if LLVM_HAS_CPP_ATTRIBUTE(clang::reinitializes) #define LLVM_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] #else #define LLVM_ATTRIBUTE_REINITIALIZES @@ -240,15 +261,13 @@ #endif /// LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements. -#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) +#if __cplusplus > 201402L && LLVM_HAS_CPP_ATTRIBUTE(fallthrough) #define LLVM_FALLTHROUGH [[fallthrough]] -#elif __has_cpp_attribute(gnu::fallthrough) +#elif LLVM_HAS_CPP_ATTRIBUTE(gnu::fallthrough) #define LLVM_FALLTHROUGH [[gnu::fallthrough]] -#elif !__cplusplus -// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious -// error when __has_cpp_attribute is given a scoped attribute in C mode. -#define LLVM_FALLTHROUGH -#elif __has_cpp_attribute(clang::fallthrough) +#elif __has_attribute(fallthrough) +#define LLVM_FALLTHROUGH __attribute__((fallthrough)) +#elif LLVM_HAS_CPP_ATTRIBUTE(clang::fallthrough) #define LLVM_FALLTHROUGH [[clang::fallthrough]] #else #define LLVM_FALLTHROUGH @@ -256,7 +275,7 @@ /// LLVM_REQUIRE_CONSTANT_INITIALIZATION - Apply this to globals to ensure that /// they are constant initialized. -#if __has_cpp_attribute(clang::require_constant_initialization) +#if LLVM_HAS_CPP_ATTRIBUTE(clang::require_constant_initialization) #define LLVM_REQUIRE_CONSTANT_INITIALIZATION \ [[clang::require_constant_initialization]] #else @@ -338,14 +357,6 @@ # define LLVM_ASSUME_ALIGNED(p, a) (p) #endif -/// \macro LLVM_ALIGNAS -/// Used to specify a minimum alignment for a structure or variable. -#if __GNUC__ && !__has_feature(cxx_alignas) && !LLVM_GNUC_PREREQ(4, 8, 1) -# define LLVM_ALIGNAS(x) __attribute__((aligned(x))) -#else -# define LLVM_ALIGNAS(x) alignas(x) -#endif - /// \macro LLVM_PACKED /// Used to specify a packed structure. /// LLVM_PACKED( @@ -376,8 +387,8 @@ /// \macro LLVM_PTR_SIZE /// A constant integer equivalent to the value of sizeof(void*). -/// Generally used in combination with LLVM_ALIGNAS or when doing computation in -/// the preprocessor. +/// Generally used in combination with alignas or when doing computation in the +/// preprocessor. #ifdef __SIZEOF_POINTER__ # define LLVM_PTR_SIZE __SIZEOF_POINTER__ #elif defined(_WIN64) @@ -527,6 +538,7 @@ void AnnotateIgnoreWritesEnd(const char *file, int line); #define LLVM_ENABLE_EXCEPTIONS 1 #endif +#ifdef __cplusplus namespace llvm { /// Allocate a buffer of memory with the given size and alignment. @@ -569,4 +581,5 @@ inline void deallocate_buffer(void *Ptr, size_t Size, size_t Alignment) { } // End namespace llvm +#endif // __cplusplus #endif diff --git a/include/llvm/Support/DataExtractor.h b/include/llvm/Support/DataExtractor.h index 6b08a2a2a445..f590a1e104fb 100644 --- a/include/llvm/Support/DataExtractor.h +++ b/include/llvm/Support/DataExtractor.h @@ -11,6 +11,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/Error.h" namespace llvm { @@ -42,6 +43,38 @@ class DataExtractor { uint8_t IsLittleEndian; uint8_t AddressSize; public: + /// A class representing a position in a DataExtractor, as well as any error + /// encountered during extraction. It enables one to extract a sequence of + /// values without error-checking and then checking for errors in bulk at the + /// end. The class holds an Error object, so failing to check the result of + /// the parse will result in a runtime error. The error flag is sticky and + /// will cause all subsequent extraction functions to fail without even + /// attempting to parse and without updating the Cursor offset. After clearing + /// the error flag, one can again use the Cursor object for parsing. + class Cursor { + uint64_t Offset; + Error Err; + + friend class DataExtractor; + + public: + /// Construct a cursor for extraction from the given offset. + explicit Cursor(uint64_t Offset) : Offset(Offset), Err(Error::success()) {} + + /// Checks whether the cursor is valid (i.e. no errors were encountered). In + /// case of errors, this does not clear the error flag -- one must call + /// takeError() instead. + explicit operator bool() { return !Err; } + + /// Return the current position of this Cursor. In the error state this is + /// the position of the Cursor before the first error was encountered. + uint64_t tell() const { return Offset; } + + /// Return error contained inside this Cursor, if any. Clears the internal + /// Cursor state. + Error takeError() { return std::move(Err); } + }; + /// Construct with a buffer that is owned by the caller. /// /// This constructor allows us to use data that is owned by the @@ -49,6 +82,11 @@ public: /// valid. DataExtractor(StringRef Data, bool IsLittleEndian, uint8_t AddressSize) : Data(Data), IsLittleEndian(IsLittleEndian), AddressSize(AddressSize) {} + DataExtractor(ArrayRef<uint8_t> Data, bool IsLittleEndian, + uint8_t AddressSize) + : Data(StringRef(reinterpret_cast<const char *>(Data.data()), + Data.size())), + IsLittleEndian(IsLittleEndian), AddressSize(AddressSize) {} /// Get the data pointed to by this extractor. StringRef getData() const { return Data; } @@ -79,17 +117,17 @@ public: /// pointed to by \a offset_ptr is out of bounds, or if the /// offset plus the length of the C string is out of bounds, /// NULL will be returned. - const char *getCStr(uint32_t *offset_ptr) const; + const char *getCStr(uint64_t *offset_ptr) const; - /// Extract a C string from \a *OffsetPtr. + /// Extract a C string from \a *offset_ptr. /// /// Returns a StringRef for the C String from the data at the offset - /// pointed to by \a OffsetPtr. A variable length NULL terminated C - /// string will be extracted and the \a OffsetPtr will be + /// pointed to by \a offset_ptr. A variable length NULL terminated C + /// string will be extracted and the \a offset_ptr will be /// updated with the offset of the byte that follows the NULL /// terminator byte. /// - /// \param[in,out] OffsetPtr + /// \param[in,out] offset_ptr /// A pointer to an offset within the data that will be advanced /// by the appropriate number of bytes if the value is extracted /// correctly. If the offset is out of bounds or there are not @@ -98,10 +136,10 @@ public: /// /// \return /// A StringRef for the C string value in the data. If the offset - /// pointed to by \a OffsetPtr is out of bounds, or if the + /// pointed to by \a offset_ptr is out of bounds, or if the /// offset plus the length of the C string is out of bounds, /// a default-initialized StringRef will be returned. - StringRef getCStrRef(uint32_t *OffsetPtr) const; + StringRef getCStrRef(uint64_t *offset_ptr) const; /// Extract an unsigned integer of size \a byte_size from \a /// *offset_ptr. @@ -124,10 +162,24 @@ public: /// @param[in] byte_size /// The size in byte of the integer to extract. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The unsigned integer value that was extracted, or zero on /// failure. - uint64_t getUnsigned(uint32_t *offset_ptr, uint32_t byte_size) const; + uint64_t getUnsigned(uint64_t *offset_ptr, uint32_t byte_size, + Error *Err = nullptr) const; + + /// Extract an unsigned integer of the given size from the location given by + /// the cursor. In case of an extraction error, or if the cursor is already in + /// an error state, zero is returned. + uint64_t getUnsigned(Cursor &C, uint32_t Size) const { + return getUnsigned(&C.Offset, Size, &C.Err); + } /// Extract an signed integer of size \a byte_size from \a *offset_ptr. /// @@ -152,7 +204,7 @@ public: /// @return /// The sign extended signed integer value that was extracted, /// or zero on failure. - int64_t getSigned(uint32_t *offset_ptr, uint32_t size) const; + int64_t getSigned(uint64_t *offset_ptr, uint32_t size) const; //------------------------------------------------------------------ /// Extract an pointer from \a *offset_ptr. @@ -171,10 +223,15 @@ public: /// /// @return /// The extracted pointer value as a 64 integer. - uint64_t getAddress(uint32_t *offset_ptr) const { + uint64_t getAddress(uint64_t *offset_ptr) const { return getUnsigned(offset_ptr, AddressSize); } + /// Extract a pointer-sized unsigned integer from the location given by the + /// cursor. In case of an extraction error, or if the cursor is already in + /// an error state, zero is returned. + uint64_t getAddress(Cursor &C) const { return getUnsigned(C, AddressSize); } + /// Extract a uint8_t value from \a *offset_ptr. /// /// Extract a single uint8_t from the binary data at the offset @@ -187,9 +244,20 @@ public: /// enough bytes to extract this value, the offset will be left /// unmodified. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The extracted uint8_t value. - uint8_t getU8(uint32_t *offset_ptr) const; + uint8_t getU8(uint64_t *offset_ptr, Error *Err = nullptr) const; + + /// Extract a single uint8_t value from the location given by the cursor. In + /// case of an extraction error, or if the cursor is already in an error + /// state, zero is returned. + uint8_t getU8(Cursor &C) const { return getU8(&C.Offset, &C.Err); } /// Extract \a count uint8_t values from \a *offset_ptr. /// @@ -214,7 +282,27 @@ public: /// @return /// \a dst if all values were properly extracted and copied, /// NULL otherise. - uint8_t *getU8(uint32_t *offset_ptr, uint8_t *dst, uint32_t count) const; + uint8_t *getU8(uint64_t *offset_ptr, uint8_t *dst, uint32_t count) const; + + /// Extract \a Count uint8_t values from the location given by the cursor and + /// store them into the destination buffer. In case of an extraction error, or + /// if the cursor is already in an error state, a nullptr is returned and the + /// destination buffer is left unchanged. + uint8_t *getU8(Cursor &C, uint8_t *Dst, uint32_t Count) const; + + /// Extract \a Count uint8_t values from the location given by the cursor and + /// store them into the destination vector. The vector is resized to fit the + /// extracted data. In case of an extraction error, or if the cursor is + /// already in an error state, the destination vector is left unchanged and + /// cursor is placed into an error state. + void getU8(Cursor &C, SmallVectorImpl<uint8_t> &Dst, uint32_t Count) const { + if (isValidOffsetForDataOfSize(C.Offset, Count)) + Dst.resize(Count); + + // This relies on the fact that getU8 will not attempt to write to the + // buffer if isValidOffsetForDataOfSize(C.Offset, Count) is false. + getU8(C, Dst.data(), Count); + } //------------------------------------------------------------------ /// Extract a uint16_t value from \a *offset_ptr. @@ -229,10 +317,21 @@ public: /// enough bytes to extract this value, the offset will be left /// unmodified. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The extracted uint16_t value. //------------------------------------------------------------------ - uint16_t getU16(uint32_t *offset_ptr) const; + uint16_t getU16(uint64_t *offset_ptr, Error *Err = nullptr) const; + + /// Extract a single uint16_t value from the location given by the cursor. In + /// case of an extraction error, or if the cursor is already in an error + /// state, zero is returned. + uint16_t getU16(Cursor &C) const { return getU16(&C.Offset, &C.Err); } /// Extract \a count uint16_t values from \a *offset_ptr. /// @@ -257,7 +356,7 @@ public: /// @return /// \a dst if all values were properly extracted and copied, /// NULL otherise. - uint16_t *getU16(uint32_t *offset_ptr, uint16_t *dst, uint32_t count) const; + uint16_t *getU16(uint64_t *offset_ptr, uint16_t *dst, uint32_t count) const; /// Extract a 24-bit unsigned value from \a *offset_ptr and return it /// in a uint32_t. @@ -274,7 +373,7 @@ public: /// /// @return /// The extracted 24-bit value represented in a uint32_t. - uint32_t getU24(uint32_t *offset_ptr) const; + uint32_t getU24(uint64_t *offset_ptr) const; /// Extract a uint32_t value from \a *offset_ptr. /// @@ -288,9 +387,20 @@ public: /// enough bytes to extract this value, the offset will be left /// unmodified. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The extracted uint32_t value. - uint32_t getU32(uint32_t *offset_ptr) const; + uint32_t getU32(uint64_t *offset_ptr, Error *Err = nullptr) const; + + /// Extract a single uint32_t value from the location given by the cursor. In + /// case of an extraction error, or if the cursor is already in an error + /// state, zero is returned. + uint32_t getU32(Cursor &C) const { return getU32(&C.Offset, &C.Err); } /// Extract \a count uint32_t values from \a *offset_ptr. /// @@ -315,7 +425,7 @@ public: /// @return /// \a dst if all values were properly extracted and copied, /// NULL otherise. - uint32_t *getU32(uint32_t *offset_ptr, uint32_t *dst, uint32_t count) const; + uint32_t *getU32(uint64_t *offset_ptr, uint32_t *dst, uint32_t count) const; /// Extract a uint64_t value from \a *offset_ptr. /// @@ -329,9 +439,20 @@ public: /// enough bytes to extract this value, the offset will be left /// unmodified. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The extracted uint64_t value. - uint64_t getU64(uint32_t *offset_ptr) const; + uint64_t getU64(uint64_t *offset_ptr, Error *Err = nullptr) const; + + /// Extract a single uint64_t value from the location given by the cursor. In + /// case of an extraction error, or if the cursor is already in an error + /// state, zero is returned. + uint64_t getU64(Cursor &C) const { return getU64(&C.Offset, &C.Err); } /// Extract \a count uint64_t values from \a *offset_ptr. /// @@ -356,7 +477,7 @@ public: /// @return /// \a dst if all values were properly extracted and copied, /// NULL otherise. - uint64_t *getU64(uint32_t *offset_ptr, uint64_t *dst, uint32_t count) const; + uint64_t *getU64(uint64_t *offset_ptr, uint64_t *dst, uint32_t count) const; /// Extract a signed LEB128 value from \a *offset_ptr. /// @@ -374,7 +495,7 @@ public: /// /// @return /// The extracted signed integer value. - int64_t getSLEB128(uint32_t *offset_ptr) const; + int64_t getSLEB128(uint64_t *offset_ptr) const; /// Extract a unsigned LEB128 value from \a *offset_ptr. /// @@ -390,23 +511,44 @@ public: /// enough bytes to extract this value, the offset will be left /// unmodified. /// + /// @param[in,out] Err + /// A pointer to an Error object. Upon return the Error object is set to + /// indicate the result (success/failure) of the function. If the Error + /// object is already set when calling this function, no extraction is + /// performed. + /// /// @return /// The extracted unsigned integer value. - uint64_t getULEB128(uint32_t *offset_ptr) const; + uint64_t getULEB128(uint64_t *offset_ptr, llvm::Error *Err = nullptr) const; + + /// Extract an unsigned ULEB128 value from the location given by the cursor. + /// In case of an extraction error, or if the cursor is already in an error + /// state, zero is returned. + uint64_t getULEB128(Cursor &C) const { return getULEB128(&C.Offset, &C.Err); } + + /// Advance the Cursor position by the given number of bytes. No-op if the + /// cursor is in an error state. + void skip(Cursor &C, uint64_t Length) const; + + /// Return true iff the cursor is at the end of the buffer, regardless of the + /// error state of the cursor. The only way both eof and error states can be + /// true is if one attempts a read while the cursor is at the very end of the + /// data buffer. + bool eof(const Cursor &C) const { return Data.size() == C.Offset; } /// Test the validity of \a offset. /// /// @return /// \b true if \a offset is a valid offset into the data in this /// object, \b false otherwise. - bool isValidOffset(uint32_t offset) const { return Data.size() > offset; } + bool isValidOffset(uint64_t offset) const { return Data.size() > offset; } /// Test the availability of \a length bytes of data from \a offset. /// /// @return /// \b true if \a offset is a valid offset and there are \a /// length bytes available at that offset, \b false otherwise. - bool isValidOffsetForDataOfSize(uint32_t offset, uint32_t length) const { + bool isValidOffsetForDataOfSize(uint64_t offset, uint64_t length) const { return offset + length >= offset && isValidOffset(offset + length - 1); } @@ -417,9 +559,15 @@ public: /// \b true if \a offset is a valid offset and there are enough /// bytes for a pointer available at that offset, \b false /// otherwise. - bool isValidOffsetForAddress(uint32_t offset) const { + bool isValidOffsetForAddress(uint64_t offset) const { return isValidOffsetForDataOfSize(offset, AddressSize); } + +protected: + // Make it possible for subclasses to access these fields without making them + // public. + static uint64_t &getOffset(Cursor &C) { return C.Offset; } + static Error &getError(Cursor &C) { return C.Err; } }; } // namespace llvm diff --git a/include/llvm/Support/Endian.h b/include/llvm/Support/Endian.h index d8be94427d7e..87aecedd3a4b 100644 --- a/include/llvm/Support/Endian.h +++ b/include/llvm/Support/Endian.h @@ -203,9 +203,8 @@ inline void writeAtBitAlignment(void *memory, value_type value, namespace detail { -template<typename ValueType, - endianness Endian, - std::size_t Alignment> +template <typename ValueType, endianness Endian, std::size_t Alignment, + std::size_t ALIGN = PickAlignment<ValueType, Alignment>::value> struct packed_endian_specific_integral { using value_type = ValueType; static constexpr endianness endian = Endian; @@ -246,8 +245,9 @@ struct packed_endian_specific_integral { } private: - AlignedCharArray<PickAlignment<value_type, alignment>::value, - sizeof(value_type)> Value; + struct { + alignas(ALIGN) char buffer[sizeof(value_type)]; + } Value; public: struct ref { diff --git a/include/llvm/Support/Error.h b/include/llvm/Support/Error.h index 299fce7a1368..350877a219bf 100644 --- a/include/llvm/Support/Error.h +++ b/include/llvm/Support/Error.h @@ -328,7 +328,7 @@ inline ErrorSuccess Error::success() { return ErrorSuccess(); } /// Make a Error instance representing failure using the given error info /// type. template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&... Args) { - return Error(llvm::make_unique<ErrT>(std::forward<ArgTs>(Args)...)); + return Error(std::make_unique<ErrT>(std::forward<ArgTs>(Args)...)); } /// Base class for user error types. Users should declare their error types @@ -548,7 +548,7 @@ public: /// Take ownership of the stored error. /// After calling this the Expected<T> is in an indeterminate state that can /// only be safely destructed. No further calls (beside the destructor) should - /// be made on the Expected<T> vaule. + /// be made on the Expected<T> value. Error takeError() { #if LLVM_ENABLE_ABI_BREAKING_CHECKS Unchecked = false; @@ -704,6 +704,12 @@ inline void cantFail(Error Err, const char *Msg = nullptr) { if (Err) { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; +#ifndef NDEBUG + std::string Str; + raw_string_ostream OS(Str); + OS << Msg << "\n" << Err; + Msg = OS.str().c_str(); +#endif llvm_unreachable(Msg); } } @@ -728,6 +734,13 @@ T cantFail(Expected<T> ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; +#ifndef NDEBUG + std::string Str; + raw_string_ostream OS(Str); + auto E = ValOrErr.takeError(); + OS << Msg << "\n" << E; + Msg = OS.str().c_str(); +#endif llvm_unreachable(Msg); } } @@ -752,6 +765,13 @@ T& cantFail(Expected<T&> ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; +#ifndef NDEBUG + std::string Str; + raw_string_ostream OS(Str); + auto E = ValOrErr.takeError(); + OS << Msg << "\n" << E; + Msg = OS.str().c_str(); +#endif llvm_unreachable(Msg); } } @@ -982,6 +1002,20 @@ inline void consumeError(Error Err) { handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {}); } +/// Convert an Expected to an Optional without doing anything. This method +/// should be used only where an error can be considered a reasonable and +/// expected return value. +/// +/// Uses of this method are potentially indicative of problems: perhaps the +/// error should be propagated further, or the error-producer should just +/// return an Optional in the first place. +template <typename T> Optional<T> expectedToOptional(Expected<T> &&E) { + if (E) + return std::move(*E); + consumeError(E.takeError()); + return None; +} + /// Helper for converting an Error to a bool. /// /// This method returns true if Err is in an error state, or false if it is @@ -1170,6 +1204,10 @@ inline Error createStringError(std::error_code EC, char const *Fmt, Error createStringError(std::error_code EC, char const *Msg); +inline Error createStringError(std::error_code EC, const Twine &S) { + return createStringError(EC, S.str().c_str()); +} + template <typename... Ts> inline Error createStringError(std::errc EC, char const *Fmt, const Ts &... Vals) { diff --git a/include/llvm/Support/FileCheck.h b/include/llvm/Support/FileCheck.h index 0cd25a71a3b3..2547449246a8 100644 --- a/include/llvm/Support/FileCheck.h +++ b/include/llvm/Support/FileCheck.h @@ -13,12 +13,12 @@ #ifndef LLVM_SUPPORT_FILECHECK_H #define LLVM_SUPPORT_FILECHECK_H -#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" #include "llvm/Support/SourceMgr.h" +#include <string> #include <vector> -#include <map> namespace llvm { @@ -30,6 +30,7 @@ struct FileCheckRequest { std::vector<std::string> GlobalDefines; bool AllowEmptyInput = false; bool MatchFullLines = false; + bool IgnoreCase = false; bool EnableVarScope = false; bool AllowDeprecatedDagOverlap = false; bool Verbose = false; @@ -37,217 +38,7 @@ struct FileCheckRequest { }; //===----------------------------------------------------------------------===// -// Numeric substitution handling code. -//===----------------------------------------------------------------------===// - -/// Base class representing the AST of a given expression. -class FileCheckExpressionAST { -public: - virtual ~FileCheckExpressionAST() = default; - - /// Evaluates and \returns the value of the expression represented by this - /// AST or an error if evaluation fails. - virtual Expected<uint64_t> eval() const = 0; -}; - -/// Class representing an unsigned literal in the AST of an expression. -class FileCheckExpressionLiteral : public FileCheckExpressionAST { -private: - /// Actual value of the literal. - uint64_t Value; - -public: - /// Constructs a literal with the specified value. - FileCheckExpressionLiteral(uint64_t Val) : Value(Val) {} - - /// \returns the literal's value. - Expected<uint64_t> eval() const { return Value; } -}; - -/// Class to represent an undefined variable error, which quotes that -/// variable's name when printed. -class FileCheckUndefVarError : public ErrorInfo<FileCheckUndefVarError> { -private: - StringRef VarName; - -public: - static char ID; - - FileCheckUndefVarError(StringRef VarName) : VarName(VarName) {} - - StringRef getVarName() const { return VarName; } - - std::error_code convertToErrorCode() const override { - return inconvertibleErrorCode(); - } - - /// Print name of variable associated with this error. - void log(raw_ostream &OS) const override { - OS << "\""; - OS.write_escaped(VarName) << "\""; - } -}; - -/// Class representing a numeric variable and its associated current value. -class FileCheckNumericVariable { -private: - /// Name of the numeric variable. - StringRef Name; - - /// Value of numeric variable, if defined, or None otherwise. - Optional<uint64_t> Value; - - /// Line number where this variable is defined, or None if defined before - /// input is parsed. Used to determine whether a variable is defined on the - /// same line as a given use. - Optional<size_t> DefLineNumber; - -public: - /// Constructor for a variable \p Name defined at line \p DefLineNumber or - /// defined before input is parsed if DefLineNumber is None. - FileCheckNumericVariable(StringRef Name, - Optional<size_t> DefLineNumber = None) - : Name(Name), DefLineNumber(DefLineNumber) {} - - /// \returns name of this numeric variable. - StringRef getName() const { return Name; } - - /// \returns this variable's value. - Optional<uint64_t> getValue() const { return Value; } - - /// Sets value of this numeric variable, if undefined. Triggers an assertion - /// failure if the variable is actually defined. - void setValue(uint64_t Value); - - /// Clears value of this numeric variable, regardless of whether it is - /// currently defined or not. - void clearValue(); - - /// \returns the line number where this variable is defined, if any, or None - /// if defined before input is parsed. - Optional<size_t> getDefLineNumber() { return DefLineNumber; } -}; - -/// Class representing the use of a numeric variable in the AST of an -/// expression. -class FileCheckNumericVariableUse : public FileCheckExpressionAST { -private: - /// Name of the numeric variable. - StringRef Name; - - /// Pointer to the class instance for the variable this use is about. - FileCheckNumericVariable *NumericVariable; - -public: - FileCheckNumericVariableUse(StringRef Name, - FileCheckNumericVariable *NumericVariable) - : Name(Name), NumericVariable(NumericVariable) {} - - /// \returns the value of the variable referenced by this instance. - Expected<uint64_t> eval() const; -}; - -/// Type of functions evaluating a given binary operation. -using binop_eval_t = uint64_t (*)(uint64_t, uint64_t); - -/// Class representing a single binary operation in the AST of an expression. -class FileCheckASTBinop : public FileCheckExpressionAST { -private: - /// Left operand. - std::unique_ptr<FileCheckExpressionAST> LeftOperand; - - /// Right operand. - std::unique_ptr<FileCheckExpressionAST> RightOperand; - - /// Pointer to function that can evaluate this binary operation. - binop_eval_t EvalBinop; - -public: - FileCheckASTBinop(binop_eval_t EvalBinop, - std::unique_ptr<FileCheckExpressionAST> LeftOp, - std::unique_ptr<FileCheckExpressionAST> RightOp) - : EvalBinop(EvalBinop) { - LeftOperand = std::move(LeftOp); - RightOperand = std::move(RightOp); - } - - /// Evaluates the value of the binary operation represented by this AST, - /// using EvalBinop on the result of recursively evaluating the operands. - /// \returns the expression value or an error if an undefined numeric - /// variable is used in one of the operands. - Expected<uint64_t> eval() const; -}; - -class FileCheckPatternContext; - -/// Class representing a substitution to perform in the RegExStr string. -class FileCheckSubstitution { -protected: - /// Pointer to a class instance holding, among other things, the table with - /// the values of live string variables at the start of any given CHECK line. - /// Used for substituting string variables with the text they were defined - /// as. Expressions are linked to the numeric variables they use at - /// parse time and directly access the value of the numeric variable to - /// evaluate their value. - FileCheckPatternContext *Context; - - /// The string that needs to be substituted for something else. For a - /// string variable this is its name, otherwise this is the whole expression. - StringRef FromStr; - - // Index in RegExStr of where to do the substitution. - size_t InsertIdx; - -public: - FileCheckSubstitution(FileCheckPatternContext *Context, StringRef VarName, - size_t InsertIdx) - : Context(Context), FromStr(VarName), InsertIdx(InsertIdx) {} - - virtual ~FileCheckSubstitution() = default; - - /// \returns the string to be substituted for something else. - StringRef getFromString() const { return FromStr; } - - /// \returns the index where the substitution is to be performed in RegExStr. - size_t getIndex() const { return InsertIdx; } - - /// \returns a string containing the result of the substitution represented - /// by this class instance or an error if substitution failed. - virtual Expected<std::string> getResult() const = 0; -}; - -class FileCheckStringSubstitution : public FileCheckSubstitution { -public: - FileCheckStringSubstitution(FileCheckPatternContext *Context, - StringRef VarName, size_t InsertIdx) - : FileCheckSubstitution(Context, VarName, InsertIdx) {} - - /// \returns the text that the string variable in this substitution matched - /// when defined, or an error if the variable is undefined. - Expected<std::string> getResult() const override; -}; - -class FileCheckNumericSubstitution : public FileCheckSubstitution { -private: - /// Pointer to the class representing the expression whose value is to be - /// substituted. - std::unique_ptr<FileCheckExpressionAST> ExpressionAST; - -public: - FileCheckNumericSubstitution(FileCheckPatternContext *Context, StringRef Expr, - std::unique_ptr<FileCheckExpressionAST> ExprAST, - size_t InsertIdx) - : FileCheckSubstitution(Context, Expr, InsertIdx) { - ExpressionAST = std::move(ExprAST); - } - - /// \returns a string containing the result of evaluating the expression in - /// this substitution, or an error if evaluation failed. - Expected<std::string> getResult() const override; -}; - -//===----------------------------------------------------------------------===// -// Pattern handling code. +// Summary of a FileCheck diagnostic. //===----------------------------------------------------------------------===// namespace Check { @@ -291,325 +82,6 @@ public: }; } // namespace Check -struct FileCheckDiag; - -/// Class holding the FileCheckPattern global state, shared by all patterns: -/// tables holding values of variables and whether they are defined or not at -/// any given time in the matching process. -class FileCheckPatternContext { - friend class FileCheckPattern; - -private: - /// When matching a given pattern, this holds the value of all the string - /// variables defined in previous patterns. In a pattern, only the last - /// definition for a given variable is recorded in this table. - /// Back-references are used for uses after any the other definition. - StringMap<StringRef> GlobalVariableTable; - - /// Map of all string variables defined so far. Used at parse time to detect - /// a name conflict between a numeric variable and a string variable when - /// the former is defined on a later line than the latter. - StringMap<bool> DefinedVariableTable; - - /// When matching a given pattern, this holds the pointers to the classes - /// representing the numeric variables defined in previous patterns. When - /// matching a pattern all definitions for that pattern are recorded in the - /// NumericVariableDefs table in the FileCheckPattern instance of that - /// pattern. - StringMap<FileCheckNumericVariable *> GlobalNumericVariableTable; - - /// Pointer to the class instance representing the @LINE pseudo variable for - /// easily updating its value. - FileCheckNumericVariable *LineVariable = nullptr; - - /// Vector holding pointers to all parsed numeric variables. Used to - /// automatically free them once they are guaranteed to no longer be used. - std::vector<std::unique_ptr<FileCheckNumericVariable>> NumericVariables; - - /// Vector holding pointers to all substitutions. Used to automatically free - /// them once they are guaranteed to no longer be used. - std::vector<std::unique_ptr<FileCheckSubstitution>> Substitutions; - -public: - /// \returns the value of string variable \p VarName or an error if no such - /// variable has been defined. - Expected<StringRef> getPatternVarValue(StringRef VarName); - - /// Defines string and numeric variables from definitions given on the - /// command line, passed as a vector of [#]VAR=VAL strings in - /// \p CmdlineDefines. \returns an error list containing diagnostics against - /// \p SM for all definition parsing failures, if any, or Success otherwise. - Error defineCmdlineVariables(std::vector<std::string> &CmdlineDefines, - SourceMgr &SM); - - /// Create @LINE pseudo variable. Value is set when pattern are being - /// matched. - void createLineVariable(); - - /// Undefines local variables (variables whose name does not start with a '$' - /// sign), i.e. removes them from GlobalVariableTable and from - /// GlobalNumericVariableTable and also clears the value of numeric - /// variables. - void clearLocalVars(); - -private: - /// Makes a new numeric variable and registers it for destruction when the - /// context is destroyed. - template <class... Types> - FileCheckNumericVariable *makeNumericVariable(Types... args); - - /// Makes a new string substitution and registers it for destruction when the - /// context is destroyed. - FileCheckSubstitution *makeStringSubstitution(StringRef VarName, - size_t InsertIdx); - - /// Makes a new numeric substitution and registers it for destruction when - /// the context is destroyed. - FileCheckSubstitution * - makeNumericSubstitution(StringRef ExpressionStr, - std::unique_ptr<FileCheckExpressionAST> ExpressionAST, - size_t InsertIdx); -}; - -/// Class to represent an error holding a diagnostic with location information -/// used when printing it. -class FileCheckErrorDiagnostic : public ErrorInfo<FileCheckErrorDiagnostic> { -private: - SMDiagnostic Diagnostic; - -public: - static char ID; - - FileCheckErrorDiagnostic(SMDiagnostic &&Diag) : Diagnostic(Diag) {} - - std::error_code convertToErrorCode() const override { - return inconvertibleErrorCode(); - } - - /// Print diagnostic associated with this error when printing the error. - void log(raw_ostream &OS) const override { Diagnostic.print(nullptr, OS); } - - static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg) { - return make_error<FileCheckErrorDiagnostic>( - SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg)); - } - - static Error get(const SourceMgr &SM, StringRef Buffer, const Twine &ErrMsg) { - return get(SM, SMLoc::getFromPointer(Buffer.data()), ErrMsg); - } -}; - -class FileCheckNotFoundError : public ErrorInfo<FileCheckNotFoundError> { -public: - static char ID; - - std::error_code convertToErrorCode() const override { - return inconvertibleErrorCode(); - } - - /// Print diagnostic associated with this error when printing the error. - void log(raw_ostream &OS) const override { - OS << "String not found in input"; - } -}; - -class FileCheckPattern { - SMLoc PatternLoc; - - /// A fixed string to match as the pattern or empty if this pattern requires - /// a regex match. - StringRef FixedStr; - - /// A regex string to match as the pattern or empty if this pattern requires - /// a fixed string to match. - std::string RegExStr; - - /// Entries in this vector represent a substitution of a string variable or - /// an expression in the RegExStr regex at match time. For example, in the - /// case of a CHECK directive with the pattern "foo[[bar]]baz[[#N+1]]", - /// RegExStr will contain "foobaz" and we'll get two entries in this vector - /// that tells us to insert the value of string variable "bar" at offset 3 - /// and the value of expression "N+1" at offset 6. - std::vector<FileCheckSubstitution *> Substitutions; - - /// Maps names of string variables defined in a pattern to the number of - /// their parenthesis group in RegExStr capturing their last definition. - /// - /// E.g. for the pattern "foo[[bar:.*]]baz([[bar]][[QUUX]][[bar:.*]])", - /// RegExStr will be "foo(.*)baz(\1<quux value>(.*))" where <quux value> is - /// the value captured for QUUX on the earlier line where it was defined, and - /// VariableDefs will map "bar" to the third parenthesis group which captures - /// the second definition of "bar". - /// - /// Note: uses std::map rather than StringMap to be able to get the key when - /// iterating over values. - std::map<StringRef, unsigned> VariableDefs; - - /// Structure representing the definition of a numeric variable in a pattern. - /// It holds the pointer to the class representing the numeric variable whose - /// value is being defined and the number of the parenthesis group in - /// RegExStr to capture that value. - struct FileCheckNumericVariableMatch { - /// Pointer to class representing the numeric variable whose value is being - /// defined. - FileCheckNumericVariable *DefinedNumericVariable; - - /// Number of the parenthesis group in RegExStr that captures the value of - /// this numeric variable definition. - unsigned CaptureParenGroup; - }; - - /// Holds the number of the parenthesis group in RegExStr and pointer to the - /// corresponding FileCheckNumericVariable class instance of all numeric - /// variable definitions. Used to set the matched value of all those - /// variables. - StringMap<FileCheckNumericVariableMatch> NumericVariableDefs; - - /// Pointer to a class instance holding the global state shared by all - /// patterns: - /// - separate tables with the values of live string and numeric variables - /// respectively at the start of any given CHECK line; - /// - table holding whether a string variable has been defined at any given - /// point during the parsing phase. - FileCheckPatternContext *Context; - - Check::FileCheckType CheckTy; - - /// Line number for this CHECK pattern or None if it is an implicit pattern. - /// Used to determine whether a variable definition is made on an earlier - /// line to the one with this CHECK. - Optional<size_t> LineNumber; - -public: - FileCheckPattern(Check::FileCheckType Ty, FileCheckPatternContext *Context, - Optional<size_t> Line = None) - : Context(Context), CheckTy(Ty), LineNumber(Line) {} - - /// \returns the location in source code. - SMLoc getLoc() const { return PatternLoc; } - - /// \returns the pointer to the global state for all patterns in this - /// FileCheck instance. - FileCheckPatternContext *getContext() const { return Context; } - - /// \returns whether \p C is a valid first character for a variable name. - static bool isValidVarNameStart(char C); - - /// Parsing information about a variable. - struct VariableProperties { - StringRef Name; - bool IsPseudo; - }; - - /// Parses the string at the start of \p Str for a variable name. \returns - /// a VariableProperties structure holding the variable name and whether it - /// is the name of a pseudo variable, or an error holding a diagnostic - /// against \p SM if parsing fail. If parsing was successful, also strips - /// \p Str from the variable name. - static Expected<VariableProperties> parseVariable(StringRef &Str, - const SourceMgr &SM); - /// Parses \p Expr for the name of a numeric variable to be defined at line - /// \p LineNumber or before input is parsed if \p LineNumber is None. - /// \returns a pointer to the class instance representing that variable, - /// creating it if needed, or an error holding a diagnostic against \p SM - /// should defining such a variable be invalid. - static Expected<FileCheckNumericVariable *> parseNumericVariableDefinition( - StringRef &Expr, FileCheckPatternContext *Context, - Optional<size_t> LineNumber, const SourceMgr &SM); - /// Parses \p Expr for a numeric substitution block. Parameter - /// \p IsLegacyLineExpr indicates whether \p Expr should be a legacy @LINE - /// expression. \returns a pointer to the class instance representing the AST - /// of the expression whose value must be substituted, or an error holding a - /// diagnostic against \p SM if parsing fails. If substitution was - /// successful, sets \p DefinedNumericVariable to point to the class - /// representing the numeric variable being defined in this numeric - /// substitution block, or None if this block does not define any variable. - Expected<std::unique_ptr<FileCheckExpressionAST>> - parseNumericSubstitutionBlock( - StringRef Expr, - Optional<FileCheckNumericVariable *> &DefinedNumericVariable, - bool IsLegacyLineExpr, const SourceMgr &SM) const; - /// Parses the pattern in \p PatternStr and initializes this FileCheckPattern - /// instance accordingly. - /// - /// \p Prefix provides which prefix is being matched, \p Req describes the - /// global options that influence the parsing such as whitespace - /// canonicalization, \p SM provides the SourceMgr used for error reports. - /// \returns true in case of an error, false otherwise. - bool parsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM, - const FileCheckRequest &Req); - /// Matches the pattern string against the input buffer \p Buffer - /// - /// \returns the position that is matched or an error indicating why matching - /// failed. If there is a match, updates \p MatchLen with the size of the - /// matched string. - /// - /// The GlobalVariableTable StringMap in the FileCheckPatternContext class - /// instance provides the current values of FileCheck string variables and - /// is updated if this match defines new values. Likewise, the - /// GlobalNumericVariableTable StringMap in the same class provides the - /// current values of FileCheck numeric variables and is updated if this - /// match defines new numeric values. - Expected<size_t> match(StringRef Buffer, size_t &MatchLen, - const SourceMgr &SM) const; - /// Prints the value of successful substitutions or the name of the undefined - /// string or numeric variables preventing a successful substitution. - void printSubstitutions(const SourceMgr &SM, StringRef Buffer, - SMRange MatchRange = None) const; - void printFuzzyMatch(const SourceMgr &SM, StringRef Buffer, - std::vector<FileCheckDiag> *Diags) const; - - bool hasVariable() const { - return !(Substitutions.empty() && VariableDefs.empty()); - } - - Check::FileCheckType getCheckTy() const { return CheckTy; } - - int getCount() const { return CheckTy.getCount(); } - -private: - bool AddRegExToRegEx(StringRef RS, unsigned &CurParen, SourceMgr &SM); - void AddBackrefToRegEx(unsigned BackrefNum); - /// Computes an arbitrary estimate for the quality of matching this pattern - /// at the start of \p Buffer; a distance of zero should correspond to a - /// perfect match. - unsigned computeMatchDistance(StringRef Buffer) const; - /// Finds the closing sequence of a regex variable usage or definition. - /// - /// \p Str has to point in the beginning of the definition (right after the - /// opening sequence). \p SM holds the SourceMgr used for error repporting. - /// \returns the offset of the closing sequence within Str, or npos if it - /// was not found. - size_t FindRegexVarEnd(StringRef Str, SourceMgr &SM); - - /// Parses \p Name as a (pseudo if \p IsPseudo is true) numeric variable use. - /// \returns the pointer to the class instance representing that variable if - /// successful, or an error holding a diagnostic against \p SM otherwise. - Expected<std::unique_ptr<FileCheckNumericVariableUse>> - parseNumericVariableUse(StringRef Name, bool IsPseudo, - const SourceMgr &SM) const; - enum class AllowedOperand { LineVar, Literal, Any }; - /// Parses \p Expr for use of a numeric operand. Accepts both literal values - /// and numeric variables, depending on the value of \p AO. \returns the - /// class representing that operand in the AST of the expression or an error - /// holding a diagnostic against \p SM otherwise. - Expected<std::unique_ptr<FileCheckExpressionAST>> - parseNumericOperand(StringRef &Expr, AllowedOperand AO, - const SourceMgr &SM) const; - /// Parses \p Expr for a binary operation. The left operand of this binary - /// operation is given in \p LeftOp and \p IsLegacyLineExpr indicates whether - /// we are parsing a legacy @LINE expression. \returns the class representing - /// the binary operation in the AST of the expression, or an error holding a - /// diagnostic against \p SM otherwise. - Expected<std::unique_ptr<FileCheckExpressionAST>> - parseBinop(StringRef &Expr, std::unique_ptr<FileCheckExpressionAST> LeftOp, - bool IsLegacyLineExpr, const SourceMgr &SM) const; -}; - -//===----------------------------------------------------------------------===// -/// Summary of a FileCheck diagnostic. -//===----------------------------------------------------------------------===// - struct FileCheckDiag { /// What is the FileCheck directive for this diagnostic? Check::FileCheckType CheckTy; @@ -659,61 +131,20 @@ struct FileCheckDiag { SMLoc CheckLoc, MatchType MatchTy, SMRange InputRange); }; -//===----------------------------------------------------------------------===// -// Check Strings. -//===----------------------------------------------------------------------===// - -/// A check that we found in the input file. -struct FileCheckString { - /// The pattern to match. - FileCheckPattern Pat; - - /// Which prefix name this check matched. - StringRef Prefix; - - /// The location in the match file that the check string was specified. - SMLoc Loc; - - /// All of the strings that are disallowed from occurring between this match - /// string and the previous one (or start of file). - std::vector<FileCheckPattern> DagNotStrings; - - FileCheckString(const FileCheckPattern &P, StringRef S, SMLoc L) - : Pat(P), Prefix(S), Loc(L) {} - - /// Matches check string and its "not strings" and/or "dag strings". - size_t Check(const SourceMgr &SM, StringRef Buffer, bool IsLabelScanMode, - size_t &MatchLen, FileCheckRequest &Req, - std::vector<FileCheckDiag> *Diags) const; - - /// Verifies that there is a single line in the given \p Buffer. Errors are - /// reported against \p SM. - bool CheckNext(const SourceMgr &SM, StringRef Buffer) const; - /// Verifies that there is no newline in the given \p Buffer. Errors are - /// reported against \p SM. - bool CheckSame(const SourceMgr &SM, StringRef Buffer) const; - /// Verifies that none of the strings in \p NotStrings are found in the given - /// \p Buffer. Errors are reported against \p SM and diagnostics recorded in - /// \p Diags according to the verbosity level set in \p Req. - bool CheckNot(const SourceMgr &SM, StringRef Buffer, - const std::vector<const FileCheckPattern *> &NotStrings, - const FileCheckRequest &Req, - std::vector<FileCheckDiag> *Diags) const; - /// Matches "dag strings" and their mixed "not strings". - size_t CheckDag(const SourceMgr &SM, StringRef Buffer, - std::vector<const FileCheckPattern *> &NotStrings, - const FileCheckRequest &Req, - std::vector<FileCheckDiag> *Diags) const; -}; +class FileCheckPatternContext; +struct FileCheckString; /// FileCheck class takes the request and exposes various methods that /// use information from the request. class FileCheck { FileCheckRequest Req; - FileCheckPatternContext PatternContext; + std::unique_ptr<FileCheckPatternContext> PatternContext; + // C++17 TODO: make this a plain std::vector. + std::unique_ptr<std::vector<FileCheckString>> CheckStrings; public: - FileCheck(FileCheckRequest Req) : Req(Req) {} + explicit FileCheck(FileCheckRequest Req); + ~FileCheck(); // Combines the check prefixes into a single regex so that we can efficiently // scan for any of the set. @@ -723,13 +154,11 @@ public: Regex buildCheckPrefixRegex(); /// Reads the check file from \p Buffer and records the expected strings it - /// contains in the \p CheckStrings vector. Errors are reported against - /// \p SM. + /// contains. Errors are reported against \p SM. /// /// Only expected strings whose prefix is one of those listed in \p PrefixRE /// are recorded. \returns true in case of an error, false otherwise. - bool ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE, - std::vector<FileCheckString> &CheckStrings); + bool readCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE); bool ValidateCheckPrefixes(); @@ -739,13 +168,14 @@ public: SmallVectorImpl<char> &OutputBuffer); /// Checks the input to FileCheck provided in the \p Buffer against the - /// \p CheckStrings read from the check file and record diagnostics emitted + /// expected strings read from the check file and record diagnostics emitted /// in \p Diags. Errors are recorded against \p SM. /// /// \returns false if the input fails to satisfy the checks. - bool CheckInput(SourceMgr &SM, StringRef Buffer, - ArrayRef<FileCheckString> CheckStrings, + bool checkInput(SourceMgr &SM, StringRef Buffer, std::vector<FileCheckDiag> *Diags = nullptr); }; + } // namespace llvm + #endif diff --git a/include/llvm/Support/FileCollector.h b/include/llvm/Support/FileCollector.h new file mode 100644 index 000000000000..19429bd3e9b4 --- /dev/null +++ b/include/llvm/Support/FileCollector.h @@ -0,0 +1,79 @@ +//===-- FileCollector.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_FILE_COLLECTOR_H +#define LLVM_SUPPORT_FILE_COLLECTOR_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/VirtualFileSystem.h" + +#include <mutex> + +namespace llvm { + +/// Collects files into a directory and generates a mapping that can be used by +/// the VFS. +class FileCollector { +public: + FileCollector(std::string Root, std::string OverlayRoot); + + void addFile(const Twine &file); + + /// Write the yaml mapping (for the VFS) to the given file. + std::error_code writeMapping(StringRef mapping_file); + + /// Copy the files into the root directory. + /// + /// When StopOnError is true (the default) we abort as soon as one file + /// cannot be copied. This is relatively common, for example when a file was + /// removed after it was added to the mapping. + std::error_code copyFiles(bool StopOnError = true); + + /// Create a VFS that collects all the paths that might be looked at by the + /// file system accesses. + static IntrusiveRefCntPtr<vfs::FileSystem> + createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS, + std::shared_ptr<FileCollector> Collector); + +private: + void addFileImpl(StringRef SrcPath); + + bool markAsSeen(StringRef Path) { return Seen.insert(Path).second; } + + bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result); + + void addFileToMapping(StringRef VirtualPath, StringRef RealPath) { + VFSWriter.addFileMapping(VirtualPath, RealPath); + } + +protected: + /// Synchronizes adding files. + std::mutex Mutex; + + /// The root directory where files are copied. + std::string Root; + + /// The root directory where the VFS overlay lives. + std::string OverlayRoot; + + /// Tracks already seen files so they can be skipped. + StringSet<> Seen; + + /// The yaml mapping writer. + vfs::YAMLVFSWriter VFSWriter; + + /// Caches RealPath calls when resolving symlinks. + StringMap<std::string> SymlinkMap; +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_FILE_COLLECTOR_H diff --git a/include/llvm/Support/FileSystem.h b/include/llvm/Support/FileSystem.h index 1bec27bddad9..a29a9d787947 100644 --- a/include/llvm/Support/FileSystem.h +++ b/include/llvm/Support/FileSystem.h @@ -991,29 +991,27 @@ file_t getStdoutHandle(); /// Returns kInvalidFile when the stream is closed. file_t getStderrHandle(); -/// Reads \p Buf.size() bytes from \p FileHandle into \p Buf. The number of -/// bytes actually read is returned in \p BytesRead. On Unix, this is equivalent -/// to `*BytesRead = ::read(FD, Buf.data(), Buf.size())`, with error reporting. -/// BytesRead will contain zero when reaching EOF. +/// Reads \p Buf.size() bytes from \p FileHandle into \p Buf. Returns the number +/// of bytes actually read. On Unix, this is equivalent to `return ::read(FD, +/// Buf.data(), Buf.size())`, with error reporting. Returns 0 when reaching EOF. /// /// @param FileHandle File to read from. /// @param Buf Buffer to read into. -/// @param BytesRead Output parameter of the number of bytes read. -/// @returns The error, if any, or errc::success. -std::error_code readNativeFile(file_t FileHandle, MutableArrayRef<char> Buf, - size_t *BytesRead); +/// @returns The number of bytes read, or error. +Expected<size_t> readNativeFile(file_t FileHandle, MutableArrayRef<char> Buf); /// Reads \p Buf.size() bytes from \p FileHandle at offset \p Offset into \p /// Buf. If 'pread' is available, this will use that, otherwise it will use -/// 'lseek'. Bytes requested beyond the end of the file will be zero -/// initialized. +/// 'lseek'. Returns the number of bytes actually read. Returns 0 when reaching +/// EOF. /// /// @param FileHandle File to read from. /// @param Buf Buffer to read into. /// @param Offset Offset into the file at which the read should occur. -/// @returns The error, if any, or errc::success. -std::error_code readNativeFileSlice(file_t FileHandle, - MutableArrayRef<char> Buf, size_t Offset); +/// @returns The number of bytes read, or error. +Expected<size_t> readNativeFileSlice(file_t FileHandle, + MutableArrayRef<char> Buf, + uint64_t Offset); /// @brief Opens the file with the given name in a write-only or read-write /// mode, returning its open file descriptor. If the file does not exist, it @@ -1217,9 +1215,9 @@ class directory_entry { // that whole structure, callers end up paying for a stat(). // std::filesystem::directory_entry may be a better model. std::string Path; - file_type Type; // Most platforms can provide this. - bool FollowSymlinks; // Affects the behavior of status(). - basic_file_status Status; // If available. + file_type Type = file_type::type_unknown; // Most platforms can provide this. + bool FollowSymlinks = true; // Affects the behavior of status(). + basic_file_status Status; // If available. public: explicit directory_entry(const Twine &Path, bool FollowSymlinks = true, diff --git a/include/llvm/Support/FileUtilities.h b/include/llvm/Support/FileUtilities.h index 16b2206924c3..04efdced32a4 100644 --- a/include/llvm/Support/FileUtilities.h +++ b/include/llvm/Support/FileUtilities.h @@ -14,6 +14,9 @@ #ifndef LLVM_SUPPORT_FILEUTILITIES_H #define LLVM_SUPPORT_FILEUTILITIES_H +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -72,6 +75,41 @@ namespace llvm { /// will not be removed when the object is destroyed. void releaseFile() { DeleteIt = false; } }; + + enum class atomic_write_error { + failed_to_create_uniq_file = 0, + output_stream_error, + failed_to_rename_temp_file + }; + + class AtomicFileWriteError : public llvm::ErrorInfo<AtomicFileWriteError> { + public: + AtomicFileWriteError(atomic_write_error Error) : Error(Error) {} + + void log(raw_ostream &OS) const override; + + const atomic_write_error Error; + static char ID; + + private: + // Users are not expected to use error_code. + std::error_code convertToErrorCode() const override { + return llvm::inconvertibleErrorCode(); + } + }; + + // atomic_write_error + whatever the Writer can return + + /// Creates a unique file with name according to the given \p TempPathModel, + /// writes content of \p Buffer to the file and renames it to \p FinalPath. + /// + /// \returns \c AtomicFileWriteError in case of error. + llvm::Error writeFileAtomically(StringRef TempPathModel, StringRef FinalPath, + StringRef Buffer); + + llvm::Error + writeFileAtomically(StringRef TempPathModel, StringRef FinalPath, + std::function<llvm::Error(llvm::raw_ostream &)> Writer); } // End llvm namespace #endif diff --git a/include/llvm/Support/Format.h b/include/llvm/Support/Format.h index 77dcbaebf1a3..9dd7b401b46a 100644 --- a/include/llvm/Support/Format.h +++ b/include/llvm/Support/Format.h @@ -29,6 +29,7 @@ #include <cassert> #include <cstdio> #include <tuple> +#include <utility> namespace llvm { @@ -91,7 +92,7 @@ class format_object final : public format_object_base { template <std::size_t... Is> int snprint_tuple(char *Buffer, unsigned BufferSize, - index_sequence<Is...>) const { + std::index_sequence<Is...>) const { #ifdef _MSC_VER return _snprintf(Buffer, BufferSize, Fmt, std::get<Is>(Vals)...); #else @@ -106,7 +107,7 @@ public: } int snprint(char *Buffer, unsigned BufferSize) const override { - return snprint_tuple(Buffer, BufferSize, index_sequence_for<Ts...>()); + return snprint_tuple(Buffer, BufferSize, std::index_sequence_for<Ts...>()); } }; diff --git a/include/llvm/Support/GenericDomTree.h b/include/llvm/Support/GenericDomTree.h index 99620802505b..9169379f746d 100644 --- a/include/llvm/Support/GenericDomTree.h +++ b/include/llvm/Support/GenericDomTree.h @@ -242,7 +242,7 @@ protected: using DomTreeNodeMapType = DenseMap<NodeT *, std::unique_ptr<DomTreeNodeBase<NodeT>>>; DomTreeNodeMapType DomTreeNodes; - DomTreeNodeBase<NodeT> *RootNode; + DomTreeNodeBase<NodeT> *RootNode = nullptr; ParentPtr Parent = nullptr; mutable bool DFSInfoValid = false; @@ -571,7 +571,7 @@ protected: assert(IDomNode && "Not immediate dominator specified for block!"); DFSInfoValid = false; return (DomTreeNodes[BB] = IDomNode->addChild( - llvm::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode))).get(); + std::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode))).get(); } /// Add a new node to the forward dominator tree and make it a new root. @@ -585,7 +585,7 @@ protected: "Cannot change root of post-dominator tree"); DFSInfoValid = false; DomTreeNodeBase<NodeT> *NewNode = (DomTreeNodes[BB] = - llvm::make_unique<DomTreeNodeBase<NodeT>>(BB, nullptr)).get(); + std::make_unique<DomTreeNodeBase<NodeT>>(BB, nullptr)).get(); if (Roots.empty()) { addRoot(BB); } else { diff --git a/include/llvm/Support/GenericDomTreeConstruction.h b/include/llvm/Support/GenericDomTreeConstruction.h index ccceba881718..7c0278e8770e 100644 --- a/include/llvm/Support/GenericDomTreeConstruction.h +++ b/include/llvm/Support/GenericDomTreeConstruction.h @@ -186,7 +186,7 @@ struct SemiNCAInfo { // Add a new tree node for this NodeT, and link it as a child of // IDomNode return (DT.DomTreeNodes[BB] = IDomNode->addChild( - llvm::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode))) + std::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode))) .get(); } @@ -586,7 +586,7 @@ struct SemiNCAInfo { NodePtr Root = IsPostDom ? nullptr : DT.Roots[0]; DT.RootNode = (DT.DomTreeNodes[Root] = - llvm::make_unique<DomTreeNodeBase<NodeT>>(Root, nullptr)) + std::make_unique<DomTreeNodeBase<NodeT>>(Root, nullptr)) .get(); SNCA.attachNewSubtree(DT, DT.RootNode); } @@ -611,7 +611,7 @@ struct SemiNCAInfo { // Add a new tree node for this BasicBlock, and link it as a child of // IDomNode. DT.DomTreeNodes[W] = IDomNode->addChild( - llvm::make_unique<DomTreeNodeBase<NodeT>>(W, IDomNode)); + std::make_unique<DomTreeNodeBase<NodeT>>(W, IDomNode)); } } @@ -663,7 +663,7 @@ struct SemiNCAInfo { TreeNodePtr VirtualRoot = DT.getNode(nullptr); FromTN = (DT.DomTreeNodes[From] = VirtualRoot->addChild( - llvm::make_unique<DomTreeNodeBase<NodeT>>(From, VirtualRoot))) + std::make_unique<DomTreeNodeBase<NodeT>>(From, VirtualRoot))) .get(); DT.Roots.push_back(From); } diff --git a/include/llvm/Support/GlobPattern.h b/include/llvm/Support/GlobPattern.h index 66a4cd94c12a..0098ac65fd30 100644 --- a/include/llvm/Support/GlobPattern.h +++ b/include/llvm/Support/GlobPattern.h @@ -21,7 +21,7 @@ #include <vector> // This class represents a glob pattern. Supported metacharacters -// are "*", "?", "[<chars>]" and "[^<chars>]". +// are "*", "?", "\", "[<chars>]", "[^<chars>]", and "[!<chars>]". namespace llvm { class BitVector; template <typename T> class ArrayRef; diff --git a/include/llvm/Support/Host.h b/include/llvm/Support/Host.h index b37cc514c92e..44f543c363db 100644 --- a/include/llvm/Support/Host.h +++ b/include/llvm/Support/Host.h @@ -15,39 +15,11 @@ #include "llvm/ADT/StringMap.h" -#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) -#include <endian.h> -#elif defined(_AIX) -#include <sys/machine.h> -#elif defined(__sun) -/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */ -#include <sys/types.h> -#define BIG_ENDIAN 4321 -#define LITTLE_ENDIAN 1234 -#if defined(_BIG_ENDIAN) -#define BYTE_ORDER BIG_ENDIAN -#else -#define BYTE_ORDER LITTLE_ENDIAN -#endif -#else -#if !defined(BYTE_ORDER) && !defined(_WIN32) -#include <machine/endian.h> -#endif -#endif - #include <string> namespace llvm { namespace sys { -#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN -constexpr bool IsBigEndianHost = true; -#else -constexpr bool IsBigEndianHost = false; -#endif - - static const bool IsLittleEndianHost = !IsBigEndianHost; - /// getDefaultTargetTriple() - Return the default target triple the compiler /// has been configured to produce code for. /// diff --git a/include/llvm/Support/JamCRC.h b/include/llvm/Support/JamCRC.h deleted file mode 100644 index b6fc4e7b9b03..000000000000 --- a/include/llvm/Support/JamCRC.h +++ /dev/null @@ -1,48 +0,0 @@ -//===-- llvm/Support/JamCRC.h - Cyclic Redundancy Check ---------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file contains an implementation of JamCRC. -// -// We will use the "Rocksoft^tm Model CRC Algorithm" to describe the properties -// of this CRC: -// Width : 32 -// Poly : 04C11DB7 -// Init : FFFFFFFF -// RefIn : True -// RefOut : True -// XorOut : 00000000 -// Check : 340BC6D9 (result of CRC for "123456789") -// -// N.B. We permit flexibility of the "Init" value. Some consumers of this need -// it to be zero. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_JAMCRC_H -#define LLVM_SUPPORT_JAMCRC_H - -#include "llvm/Support/DataTypes.h" - -namespace llvm { -template <typename T> class ArrayRef; - -class JamCRC { -public: - JamCRC(uint32_t Init = 0xFFFFFFFFU) : CRC(Init) {} - - // Update the CRC calculation with Data. - void update(ArrayRef<char> Data); - - uint32_t getCRC() const { return CRC; } - -private: - uint32_t CRC; -}; -} // End of namespace llvm - -#endif diff --git a/include/llvm/Support/MachineValueType.h b/include/llvm/Support/MachineValueType.h index b94d2c4836cc..7f9f0b85c55e 100644 --- a/include/llvm/Support/MachineValueType.h +++ b/include/llvm/Support/MachineValueType.h @@ -17,6 +17,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/TypeSize.h" #include <cassert> namespace llvm { @@ -64,152 +65,162 @@ namespace llvm { v32i1 = 19, // 32 x i1 v64i1 = 20, // 64 x i1 v128i1 = 21, // 128 x i1 - v512i1 = 22, // 512 x i1 - v1024i1 = 23, // 1024 x i1 - - v1i8 = 24, // 1 x i8 - v2i8 = 25, // 2 x i8 - v4i8 = 26, // 4 x i8 - v8i8 = 27, // 8 x i8 - v16i8 = 28, // 16 x i8 - v32i8 = 29, // 32 x i8 - v64i8 = 30, // 64 x i8 - v128i8 = 31, //128 x i8 - v256i8 = 32, //256 x i8 - - v1i16 = 33, // 1 x i16 - v2i16 = 34, // 2 x i16 - v4i16 = 35, // 4 x i16 - v8i16 = 36, // 8 x i16 - v16i16 = 37, // 16 x i16 - v32i16 = 38, // 32 x i16 - v64i16 = 39, // 64 x i16 - v128i16 = 40, //128 x i16 - - v1i32 = 41, // 1 x i32 - v2i32 = 42, // 2 x i32 - v3i32 = 43, // 3 x i32 - v4i32 = 44, // 4 x i32 - v5i32 = 45, // 5 x i32 - v8i32 = 46, // 8 x i32 - v16i32 = 47, // 16 x i32 - v32i32 = 48, // 32 x i32 - v64i32 = 49, // 64 x i32 - v128i32 = 50, // 128 x i32 - v256i32 = 51, // 256 x i32 - v512i32 = 52, // 512 x i32 - v1024i32 = 53, // 1024 x i32 - v2048i32 = 54, // 2048 x i32 - - v1i64 = 55, // 1 x i64 - v2i64 = 56, // 2 x i64 - v4i64 = 57, // 4 x i64 - v8i64 = 58, // 8 x i64 - v16i64 = 59, // 16 x i64 - v32i64 = 60, // 32 x i64 - - v1i128 = 61, // 1 x i128 - - // Scalable integer types - nxv1i1 = 62, // n x 1 x i1 - nxv2i1 = 63, // n x 2 x i1 - nxv4i1 = 64, // n x 4 x i1 - nxv8i1 = 65, // n x 8 x i1 - nxv16i1 = 66, // n x 16 x i1 - nxv32i1 = 67, // n x 32 x i1 - - nxv1i8 = 68, // n x 1 x i8 - nxv2i8 = 69, // n x 2 x i8 - nxv4i8 = 70, // n x 4 x i8 - nxv8i8 = 71, // n x 8 x i8 - nxv16i8 = 72, // n x 16 x i8 - nxv32i8 = 73, // n x 32 x i8 - - nxv1i16 = 74, // n x 1 x i16 - nxv2i16 = 75, // n x 2 x i16 - nxv4i16 = 76, // n x 4 x i16 - nxv8i16 = 77, // n x 8 x i16 - nxv16i16 = 78, // n x 16 x i16 - nxv32i16 = 79, // n x 32 x i16 - - nxv1i32 = 80, // n x 1 x i32 - nxv2i32 = 81, // n x 2 x i32 - nxv4i32 = 82, // n x 4 x i32 - nxv8i32 = 83, // n x 8 x i32 - nxv16i32 = 84, // n x 16 x i32 - nxv32i32 = 85, // n x 32 x i32 - - nxv1i64 = 86, // n x 1 x i64 - nxv2i64 = 87, // n x 2 x i64 - nxv4i64 = 88, // n x 4 x i64 - nxv8i64 = 89, // n x 8 x i64 - nxv16i64 = 90, // n x 16 x i64 - nxv32i64 = 91, // n x 32 x i64 - - FIRST_INTEGER_VECTOR_VALUETYPE = v1i1, - LAST_INTEGER_VECTOR_VALUETYPE = nxv32i64, - - FIRST_INTEGER_SCALABLE_VALUETYPE = nxv1i1, - LAST_INTEGER_SCALABLE_VALUETYPE = nxv32i64, - - v2f16 = 92, // 2 x f16 - v4f16 = 93, // 4 x f16 - v8f16 = 94, // 8 x f16 - v1f32 = 95, // 1 x f32 - v2f32 = 96, // 2 x f32 - v3f32 = 97, // 3 x f32 - v4f32 = 98, // 4 x f32 - v5f32 = 99, // 5 x f32 - v8f32 = 100, // 8 x f32 - v16f32 = 101, // 16 x f32 - v32f32 = 102, // 32 x f32 - v64f32 = 103, // 64 x f32 - v128f32 = 104, // 128 x f32 - v256f32 = 105, // 256 x f32 - v512f32 = 106, // 512 x f32 - v1024f32 = 107, // 1024 x f32 - v2048f32 = 108, // 2048 x f32 - v1f64 = 109, // 1 x f64 - v2f64 = 110, // 2 x f64 - v4f64 = 111, // 4 x f64 - v8f64 = 112, // 8 x f64 - - nxv2f16 = 113, // n x 2 x f16 - nxv4f16 = 114, // n x 4 x f16 - nxv8f16 = 115, // n x 8 x f16 - nxv1f32 = 116, // n x 1 x f32 - nxv2f32 = 117, // n x 2 x f32 - nxv4f32 = 118, // n x 4 x f32 - nxv8f32 = 119, // n x 8 x f32 - nxv16f32 = 120, // n x 16 x f32 - nxv1f64 = 121, // n x 1 x f64 - nxv2f64 = 122, // n x 2 x f64 - nxv4f64 = 123, // n x 4 x f64 - nxv8f64 = 124, // n x 8 x f64 - - FIRST_FP_VECTOR_VALUETYPE = v2f16, - LAST_FP_VECTOR_VALUETYPE = nxv8f64, - - FIRST_FP_SCALABLE_VALUETYPE = nxv2f16, - LAST_FP_SCALABLE_VALUETYPE = nxv8f64, + v256i1 = 22, // 256 x i1 + v512i1 = 23, // 512 x i1 + v1024i1 = 24, // 1024 x i1 + + v1i8 = 25, // 1 x i8 + v2i8 = 26, // 2 x i8 + v4i8 = 27, // 4 x i8 + v8i8 = 28, // 8 x i8 + v16i8 = 29, // 16 x i8 + v32i8 = 30, // 32 x i8 + v64i8 = 31, // 64 x i8 + v128i8 = 32, //128 x i8 + v256i8 = 33, //256 x i8 + + v1i16 = 34, // 1 x i16 + v2i16 = 35, // 2 x i16 + v3i16 = 36, // 3 x i16 + v4i16 = 37, // 4 x i16 + v8i16 = 38, // 8 x i16 + v16i16 = 39, // 16 x i16 + v32i16 = 40, // 32 x i16 + v64i16 = 41, // 64 x i16 + v128i16 = 42, //128 x i16 + + v1i32 = 43, // 1 x i32 + v2i32 = 44, // 2 x i32 + v3i32 = 45, // 3 x i32 + v4i32 = 46, // 4 x i32 + v5i32 = 47, // 5 x i32 + v8i32 = 48, // 8 x i32 + v16i32 = 49, // 16 x i32 + v32i32 = 50, // 32 x i32 + v64i32 = 51, // 64 x i32 + v128i32 = 52, // 128 x i32 + v256i32 = 53, // 256 x i32 + v512i32 = 54, // 512 x i32 + v1024i32 = 55, // 1024 x i32 + v2048i32 = 56, // 2048 x i32 + + v1i64 = 57, // 1 x i64 + v2i64 = 58, // 2 x i64 + v4i64 = 59, // 4 x i64 + v8i64 = 60, // 8 x i64 + v16i64 = 61, // 16 x i64 + v32i64 = 62, // 32 x i64 + + v1i128 = 63, // 1 x i128 + + FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i1, + LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i128, + + v2f16 = 64, // 2 x f16 + v3f16 = 65, // 3 x f16 + v4f16 = 66, // 4 x f16 + v8f16 = 67, // 8 x f16 + v16f16 = 68, // 16 x f16 + v32f16 = 69, // 32 x f16 + v1f32 = 70, // 1 x f32 + v2f32 = 71, // 2 x f32 + v3f32 = 72, // 3 x f32 + v4f32 = 73, // 4 x f32 + v5f32 = 74, // 5 x f32 + v8f32 = 75, // 8 x f32 + v16f32 = 76, // 16 x f32 + v32f32 = 77, // 32 x f32 + v64f32 = 78, // 64 x f32 + v128f32 = 79, // 128 x f32 + v256f32 = 80, // 256 x f32 + v512f32 = 81, // 512 x f32 + v1024f32 = 82, // 1024 x f32 + v2048f32 = 83, // 2048 x f32 + v1f64 = 84, // 1 x f64 + v2f64 = 85, // 2 x f64 + v4f64 = 86, // 4 x f64 + v8f64 = 87, // 8 x f64 + + FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v2f16, + LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v8f64, + + FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1, + LAST_FIXEDLEN_VECTOR_VALUETYPE = v8f64, + + nxv1i1 = 88, // n x 1 x i1 + nxv2i1 = 89, // n x 2 x i1 + nxv4i1 = 90, // n x 4 x i1 + nxv8i1 = 91, // n x 8 x i1 + nxv16i1 = 92, // n x 16 x i1 + nxv32i1 = 93, // n x 32 x i1 + + nxv1i8 = 94, // n x 1 x i8 + nxv2i8 = 95, // n x 2 x i8 + nxv4i8 = 96, // n x 4 x i8 + nxv8i8 = 97, // n x 8 x i8 + nxv16i8 = 98, // n x 16 x i8 + nxv32i8 = 99, // n x 32 x i8 + + nxv1i16 = 100, // n x 1 x i16 + nxv2i16 = 101, // n x 2 x i16 + nxv4i16 = 102, // n x 4 x i16 + nxv8i16 = 103, // n x 8 x i16 + nxv16i16 = 104, // n x 16 x i16 + nxv32i16 = 105, // n x 32 x i16 + + nxv1i32 = 106, // n x 1 x i32 + nxv2i32 = 107, // n x 2 x i32 + nxv4i32 = 108, // n x 4 x i32 + nxv8i32 = 109, // n x 8 x i32 + nxv16i32 = 110, // n x 16 x i32 + nxv32i32 = 111, // n x 32 x i32 + + nxv1i64 = 112, // n x 1 x i64 + nxv2i64 = 113, // n x 2 x i64 + nxv4i64 = 114, // n x 4 x i64 + nxv8i64 = 115, // n x 8 x i64 + nxv16i64 = 116, // n x 16 x i64 + nxv32i64 = 117, // n x 32 x i64 + + FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1, + LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64, + + nxv2f16 = 118, // n x 2 x f16 + nxv4f16 = 119, // n x 4 x f16 + nxv8f16 = 120, // n x 8 x f16 + nxv1f32 = 121, // n x 1 x f32 + nxv2f32 = 122, // n x 2 x f32 + nxv4f32 = 123, // n x 4 x f32 + nxv8f32 = 124, // n x 8 x f32 + nxv16f32 = 125, // n x 16 x f32 + nxv1f64 = 126, // n x 1 x f64 + nxv2f64 = 127, // n x 2 x f64 + nxv4f64 = 128, // n x 4 x f64 + nxv8f64 = 129, // n x 8 x f64 + + FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv2f16, + LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64, + + FIRST_SCALABLE_VECTOR_VALUETYPE = nxv1i1, + LAST_SCALABLE_VECTOR_VALUETYPE = nxv8f64, FIRST_VECTOR_VALUETYPE = v1i1, LAST_VECTOR_VALUETYPE = nxv8f64, - x86mmx = 125, // This is an X86 MMX value + x86mmx = 130, // This is an X86 MMX value - Glue = 126, // This glues nodes together during pre-RA sched + Glue = 131, // This glues nodes together during pre-RA sched - isVoid = 127, // This has no value + isVoid = 132, // This has no value - Untyped = 128, // This value takes a register, but has + Untyped = 133, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. - exnref = 129, // WebAssembly's exnref type + exnref = 134, // WebAssembly's exnref type FIRST_VALUETYPE = 1, // This is always the beginning of the list. - LAST_VALUETYPE = 130, // This always remains at the end of the list. + LAST_VALUETYPE = 135, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors @@ -253,41 +264,6 @@ namespace llvm { SimpleValueType SimpleTy = INVALID_SIMPLE_VALUE_TYPE; - // A class to represent the number of elements in a vector - // - // For fixed-length vectors, the total number of elements is equal to 'Min' - // For scalable vectors, the total number of elements is a multiple of 'Min' - class ElementCount { - public: - unsigned Min; - bool Scalable; - - ElementCount(unsigned Min, bool Scalable) - : Min(Min), Scalable(Scalable) {} - - ElementCount operator*(unsigned RHS) { - return { Min * RHS, Scalable }; - } - - ElementCount& operator*=(unsigned RHS) { - Min *= RHS; - return *this; - } - - ElementCount operator/(unsigned RHS) { - return { Min / RHS, Scalable }; - } - - ElementCount& operator/=(unsigned RHS) { - Min /= RHS; - return *this; - } - - bool operator==(const ElementCount& RHS) { - return Min == RHS.Min && Scalable == RHS.Scalable; - } - }; - constexpr MVT() = default; constexpr MVT(SimpleValueType SVT) : SimpleTy(SVT) {} @@ -308,16 +284,20 @@ namespace llvm { bool isFloatingPoint() const { return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE && SimpleTy <= MVT::LAST_FP_VALUETYPE) || - (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE && - SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE)); + (SimpleTy >= MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE) || + (SimpleTy >= MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE)); } /// Return true if this is an integer or a vector integer type. bool isInteger() const { return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE && SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) || - (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE && - SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE)); + (SimpleTy >= MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE) || + (SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE)); } /// Return true if this is an integer, not including vectors. @@ -335,10 +315,13 @@ namespace llvm { /// Return true if this is a vector value type where the /// runtime length is machine dependent bool isScalableVector() const { - return ((SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VALUETYPE && - SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VALUETYPE) || - (SimpleTy >= MVT::FIRST_FP_SCALABLE_VALUETYPE && - SimpleTy <= MVT::LAST_FP_SCALABLE_VALUETYPE)); + return (SimpleTy >= MVT::FIRST_SCALABLE_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_SCALABLE_VECTOR_VALUETYPE); + } + + bool isFixedLengthVector() const { + return (SimpleTy >= MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE); } /// Return true if this is a 16-bit vector type. @@ -373,17 +356,18 @@ namespace llvm { /// Return true if this is a 256-bit vector type. bool is256BitVector() const { - return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 || - SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 || - SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64); + return (SimpleTy == MVT::v16f16 || SimpleTy == MVT::v8f32 || + SimpleTy == MVT::v4f64 || SimpleTy == MVT::v32i8 || + SimpleTy == MVT::v16i16 || SimpleTy == MVT::v8i32 || + SimpleTy == MVT::v4i64 || SimpleTy == MVT::v256i1); } /// Return true if this is a 512-bit vector type. bool is512BitVector() const { - return (SimpleTy == MVT::v16f32 || SimpleTy == MVT::v8f64 || - SimpleTy == MVT::v512i1 || SimpleTy == MVT::v64i8 || - SimpleTy == MVT::v32i16 || SimpleTy == MVT::v16i32 || - SimpleTy == MVT::v8i64); + return (SimpleTy == MVT::v32f16 || SimpleTy == MVT::v16f32 || + SimpleTy == MVT::v8f64 || SimpleTy == MVT::v512i1 || + SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 || + SimpleTy == MVT::v16i32 || SimpleTy == MVT::v8i64); } /// Return true if this is a 1024-bit vector type. @@ -406,6 +390,15 @@ namespace llvm { SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny); } + /// Return a VT for a vector type with the same element type but + /// half the number of elements. + MVT getHalfNumVectorElementsVT() const { + MVT EltVT = getVectorElementType(); + auto EltCnt = getVectorElementCount(); + assert(!(EltCnt.Min & 1) && "Splitting vector, but not in half!"); + return getVectorVT(EltVT, EltCnt / 2); + } + /// Returns true if the given vector is a power of 2. bool isPow2VectorType() const { unsigned NElts = getVectorNumElements(); @@ -440,6 +433,7 @@ namespace llvm { case v32i1: case v64i1: case v128i1: + case v256i1: case v512i1: case v1024i1: case nxv1i1: @@ -465,6 +459,7 @@ namespace llvm { case nxv32i8: return i8; case v1i16: case v2i16: + case v3i16: case v4i16: case v8i16: case v16i16: @@ -511,8 +506,11 @@ namespace llvm { case nxv32i64: return i64; case v1i128: return i128; case v2f16: + case v3f16: case v4f16: case v8f16: + case v16f16: + case v32f16: case nxv2f16: case nxv4f16: case nxv8f16: return f16; @@ -558,6 +556,7 @@ namespace llvm { case v512i1: case v512i32: case v512f32: return 512; + case v256i1: case v256i8: case v256i32: case v256f32: return 256; @@ -576,6 +575,7 @@ namespace llvm { case v32i16: case v32i32: case v32i64: + case v32f16: case v32f32: case nxv32i1: case nxv32i8: @@ -587,6 +587,7 @@ namespace llvm { case v16i16: case v16i32: case v16i64: + case v16f16: case v16f32: case nxv16i1: case nxv16i8: @@ -628,7 +629,9 @@ namespace llvm { case nxv4f16: case nxv4f32: case nxv4f64: return 4; + case v3i16: case v3i32: + case v3f16: case v3f32: return 3; case v2i1: case v2i8: @@ -664,7 +667,7 @@ namespace llvm { } } - MVT::ElementCount getVectorElementCount() const { + ElementCount getVectorElementCount() const { return { getVectorNumElements(), isScalableVector() }; } @@ -721,6 +724,8 @@ namespace llvm { case nxv1i32: case nxv2f16: case nxv1f32: return 32; + case v3i16: + case v3f16: return 48; case x86mmx: case f64 : case i64 : @@ -763,10 +768,12 @@ namespace llvm { case nxv2f64: return 128; case v5i32: case v5f32: return 160; + case v256i1: case v32i8: case v16i16: case v8i32: case v4i64: + case v16f16: case v8f32: case v4f64: case nxv32i8: @@ -780,6 +787,7 @@ namespace llvm { case v32i16: case v16i32: case v8i64: + case v32f16: case v16f32: case v8f64: case nxv32i16: @@ -900,6 +908,7 @@ namespace llvm { if (NumElements == 32) return MVT::v32i1; if (NumElements == 64) return MVT::v64i1; if (NumElements == 128) return MVT::v128i1; + if (NumElements == 256) return MVT::v256i1; if (NumElements == 512) return MVT::v512i1; if (NumElements == 1024) return MVT::v1024i1; break; @@ -917,6 +926,7 @@ namespace llvm { case MVT::i16: if (NumElements == 1) return MVT::v1i16; if (NumElements == 2) return MVT::v2i16; + if (NumElements == 3) return MVT::v3i16; if (NumElements == 4) return MVT::v4i16; if (NumElements == 8) return MVT::v8i16; if (NumElements == 16) return MVT::v16i16; @@ -953,8 +963,11 @@ namespace llvm { break; case MVT::f16: if (NumElements == 2) return MVT::v2f16; + if (NumElements == 3) return MVT::v3f16; if (NumElements == 4) return MVT::v4f16; if (NumElements == 8) return MVT::v8f16; + if (NumElements == 16) return MVT::v16f16; + if (NumElements == 32) return MVT::v32f16; break; case MVT::f32: if (NumElements == 1) return MVT::v1f32; @@ -1054,7 +1067,7 @@ namespace llvm { return getVectorVT(VT, NumElements); } - static MVT getVectorVT(MVT VT, MVT::ElementCount EC) { + static MVT getVectorVT(MVT VT, ElementCount EC) { if (EC.Scalable) return getScalableVectorVT(VT, EC.Min); return getVectorVT(VT, EC.Min); @@ -1108,26 +1121,40 @@ namespace llvm { (MVT::SimpleValueType)(MVT::LAST_VECTOR_VALUETYPE + 1)); } - static mvt_range integer_vector_valuetypes() { + static mvt_range fixedlen_vector_valuetypes() { return mvt_range( - MVT::FIRST_INTEGER_VECTOR_VALUETYPE, - (MVT::SimpleValueType)(MVT::LAST_INTEGER_VECTOR_VALUETYPE + 1)); + MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE + 1)); } - static mvt_range fp_vector_valuetypes() { + static mvt_range scalable_vector_valuetypes() { return mvt_range( - MVT::FIRST_FP_VECTOR_VALUETYPE, - (MVT::SimpleValueType)(MVT::LAST_FP_VECTOR_VALUETYPE + 1)); + MVT::FIRST_SCALABLE_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_SCALABLE_VECTOR_VALUETYPE + 1)); + } + + static mvt_range integer_fixedlen_vector_valuetypes() { + return mvt_range( + MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE + 1)); + } + + static mvt_range fp_fixedlen_vector_valuetypes() { + return mvt_range( + MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE + 1)); } static mvt_range integer_scalable_vector_valuetypes() { - return mvt_range(MVT::FIRST_INTEGER_SCALABLE_VALUETYPE, - (MVT::SimpleValueType)(MVT::LAST_INTEGER_SCALABLE_VALUETYPE + 1)); + return mvt_range( + MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE + 1)); } static mvt_range fp_scalable_vector_valuetypes() { - return mvt_range(MVT::FIRST_FP_SCALABLE_VALUETYPE, - (MVT::SimpleValueType)(MVT::LAST_FP_SCALABLE_VALUETYPE + 1)); + return mvt_range( + MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE + 1)); } /// @} }; diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h index 249139e824b5..004a6f5f6eb8 100644 --- a/include/llvm/Support/MathExtras.h +++ b/include/llvm/Support/MathExtras.h @@ -39,6 +39,7 @@ unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); #endif namespace llvm { + /// The behavior an operation has on an input of 0. enum ZeroBehavior { /// The returned value is undefined. @@ -49,6 +50,42 @@ enum ZeroBehavior { ZB_Width }; +/// Mathematical constants. +namespace numbers { +// TODO: Track C++20 std::numbers. +// TODO: Favor using the hexadecimal FP constants (requires C++17). +constexpr double e = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113 + egamma = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620 + ln2 = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162 + ln10 = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392 + log2e = 1.4426950408889634074, // (0x1.71547652b82feP+0) + log10e = .43429448190325182765, // (0x1.bcb7b1526e50eP-2) + pi = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796 + inv_pi = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541 + sqrtpi = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161 + inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197 + sqrt2 = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219 + inv_sqrt2 = .70710678118654752440, // (0x1.6a09e667f3bcdP-1) + sqrt3 = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194 + inv_sqrt3 = .57735026918962576451, // (0x1.279a74590331cP-1) + phi = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622 +constexpr float ef = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113 + egammaf = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620 + ln2f = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162 + ln10f = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392 + log2ef = 1.44269504F, // (0x1.715476P+0) + log10ef = .434294482F, // (0x1.bcb7b2P-2) + pif = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796 + inv_pif = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541 + sqrtpif = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161 + inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197 + sqrt2f = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193 + inv_sqrt2f = .707106781F, // (0x1.6a09e6P-1) + sqrt3f = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194 + inv_sqrt3f = .577350269F, // (0x1.279a74P-1) + phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622 +} // namespace numbers + namespace detail { template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter { static unsigned count(T Val, ZeroBehavior) { @@ -73,13 +110,13 @@ template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter { } }; -#if __GNUC__ >= 4 || defined(_MSC_VER) +#if defined(__GNUC__) || defined(_MSC_VER) template <typename T> struct TrailingZerosCounter<T, 4> { static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 32; -#if __has_builtin(__builtin_ctz) || LLVM_GNUC_PREREQ(4, 0, 0) +#if __has_builtin(__builtin_ctz) || defined(__GNUC__) return __builtin_ctz(Val); #elif defined(_MSC_VER) unsigned long Index; @@ -95,7 +132,7 @@ template <typename T> struct TrailingZerosCounter<T, 8> { if (ZB != ZB_Undefined && Val == 0) return 64; -#if __has_builtin(__builtin_ctzll) || LLVM_GNUC_PREREQ(4, 0, 0) +#if __has_builtin(__builtin_ctzll) || defined(__GNUC__) return __builtin_ctzll(Val); #elif defined(_MSC_VER) unsigned long Index; @@ -142,13 +179,13 @@ template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter { } }; -#if __GNUC__ >= 4 || defined(_MSC_VER) +#if defined(__GNUC__) || defined(_MSC_VER) template <typename T> struct LeadingZerosCounter<T, 4> { static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 32; -#if __has_builtin(__builtin_clz) || LLVM_GNUC_PREREQ(4, 0, 0) +#if __has_builtin(__builtin_clz) || defined(__GNUC__) return __builtin_clz(Val); #elif defined(_MSC_VER) unsigned long Index; @@ -164,7 +201,7 @@ template <typename T> struct LeadingZerosCounter<T, 8> { if (ZB != ZB_Undefined && Val == 0) return 64; -#if __has_builtin(__builtin_clzll) || LLVM_GNUC_PREREQ(4, 0, 0) +#if __has_builtin(__builtin_clzll) || defined(__GNUC__) return __builtin_clzll(Val); #elif defined(_MSC_VER) unsigned long Index; @@ -486,7 +523,7 @@ template <typename T, std::size_t SizeOfT> struct PopulationCounter { static unsigned count(T Value) { // Generic version, forward to 32 bits. static_assert(SizeOfT <= 4, "Not implemented!"); -#if __GNUC__ >= 4 +#if defined(__GNUC__) return __builtin_popcount(Value); #else uint32_t v = Value; @@ -499,7 +536,7 @@ template <typename T, std::size_t SizeOfT> struct PopulationCounter { template <typename T> struct PopulationCounter<T, 8> { static unsigned count(T Value) { -#if __GNUC__ >= 4 +#if defined(__GNUC__) return __builtin_popcountll(Value); #else uint64_t v = Value; @@ -523,6 +560,16 @@ inline unsigned countPopulation(T Value) { return detail::PopulationCounter<T, sizeof(T)>::count(Value); } +/// Compile time Log2. +/// Valid only for positive powers of two. +template <size_t kValue> constexpr inline size_t CTLog2() { + static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue), + "Value is not a valid power of 2"); + return 1 + CTLog2<kValue / 2>(); +} + +template <> constexpr inline size_t CTLog2<1>() { return 0; } + /// Return the log base 2 of the specified value. inline double Log2(double Value) { #if defined(__ANDROID_API__) && __ANDROID_API__ < 18 @@ -620,25 +667,6 @@ constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) { return (A | B) & (1 + ~(A | B)); } -/// Aligns \c Addr to \c Alignment bytes, rounding up. -/// -/// Alignment should be a power of two. This method rounds up, so -/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8. -inline uintptr_t alignAddr(const void *Addr, size_t Alignment) { - assert(Alignment && isPowerOf2_64((uint64_t)Alignment) && - "Alignment is not a power of two!"); - - assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr); - - return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1)); -} - -/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment -/// bytes, rounding up. -inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) { - return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr; -} - /// Returns the next power of two (in 64-bits) that is strictly greater than A. /// Returns zero on overflow. inline uint64_t NextPowerOf2(uint64_t A) { @@ -704,19 +732,6 @@ inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) { return alignTo(Numerator, Denominator) / Denominator; } -/// \c alignTo for contexts where a constant expression is required. -/// \sa alignTo -/// -/// \todo FIXME: remove when \c constexpr becomes really \c constexpr -template <uint64_t Align> -struct AlignTo { - static_assert(Align != 0u, "Align must be non-zero"); - template <uint64_t Value> - struct from_value { - static const uint64_t value = (Value + Align - 1) / Align * Align; - }; -}; - /// Returns the largest uint64_t less than or equal to \p Value and is /// \p Skew mod \p Align. \p Align must be non-zero inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { @@ -725,13 +740,6 @@ inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { return (Value - Skew) / Align * Align + Skew; } -/// Returns the offset to the next integer (mod 2**64) that is greater than -/// or equal to \p Value and is a multiple of \p Align. \p Align must be -/// non-zero. -inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) { - return alignTo(Value, Align) - Value; -} - /// Sign-extend the number in the bottom B bits of X to a 32-bit integer. /// Requires 0 < B <= 32. template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) { @@ -853,6 +861,91 @@ SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) { /// Use this rather than HUGE_VALF; the latter causes warnings on MSVC. extern const float huge_valf; + + +/// Add two signed integers, computing the two's complement truncated result, +/// returning true if overflow occured. +template <typename T> +typename std::enable_if<std::is_signed<T>::value, T>::type +AddOverflow(T X, T Y, T &Result) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(X, Y, &Result); +#else + // Perform the unsigned addition. + using U = typename std::make_unsigned<T>::type; + const U UX = static_cast<U>(X); + const U UY = static_cast<U>(Y); + const U UResult = UX + UY; + + // Convert to signed. + Result = static_cast<T>(UResult); + + // Adding two positive numbers should result in a positive number. + if (X > 0 && Y > 0) + return Result <= 0; + // Adding two negatives should result in a negative number. + if (X < 0 && Y < 0) + return Result >= 0; + return false; +#endif +} + +/// Subtract two signed integers, computing the two's complement truncated +/// result, returning true if an overflow ocurred. +template <typename T> +typename std::enable_if<std::is_signed<T>::value, T>::type +SubOverflow(T X, T Y, T &Result) { +#if __has_builtin(__builtin_sub_overflow) + return __builtin_sub_overflow(X, Y, &Result); +#else + // Perform the unsigned addition. + using U = typename std::make_unsigned<T>::type; + const U UX = static_cast<U>(X); + const U UY = static_cast<U>(Y); + const U UResult = UX - UY; + + // Convert to signed. + Result = static_cast<T>(UResult); + + // Subtracting a positive number from a negative results in a negative number. + if (X <= 0 && Y > 0) + return Result >= 0; + // Subtracting a negative number from a positive results in a positive number. + if (X >= 0 && Y < 0) + return Result <= 0; + return false; +#endif +} + + +/// Multiply two signed integers, computing the two's complement truncated +/// result, returning true if an overflow ocurred. +template <typename T> +typename std::enable_if<std::is_signed<T>::value, T>::type +MulOverflow(T X, T Y, T &Result) { + // Perform the unsigned multiplication on absolute values. + using U = typename std::make_unsigned<T>::type; + const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X); + const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y); + const U UResult = UX * UY; + + // Convert to signed. + const bool IsNegative = (X < 0) ^ (Y < 0); + Result = IsNegative ? (0 - UResult) : UResult; + + // If any of the args was 0, result is 0 and no overflow occurs. + if (UX == 0 || UY == 0) + return false; + + // UX and UY are in [1, 2^n], where n is the number of digits. + // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for + // positive) divided by an argument compares to the other. + if (IsNegative) + return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY; + else + return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY; +} + } // End llvm namespace #endif diff --git a/include/llvm/Support/Mutex.h b/include/llvm/Support/Mutex.h index c3abfc7a7806..1d8a0d3c87cb 100644 --- a/include/llvm/Support/Mutex.h +++ b/include/llvm/Support/Mutex.h @@ -13,97 +13,31 @@ #ifndef LLVM_SUPPORT_MUTEX_H #define LLVM_SUPPORT_MUTEX_H -#include "llvm/Config/llvm-config.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Threading.h" #include <cassert> +#include <mutex> namespace llvm { namespace sys { - /// Platform agnostic Mutex class. - class MutexImpl - { - /// @name Constructors - /// @{ - public: - - /// Initializes the lock but doesn't acquire it. if \p recursive is set - /// to false, the lock will not be recursive which makes it cheaper but - /// also more likely to deadlock (same thread can't acquire more than - /// once). - /// Default Constructor. - explicit MutexImpl(bool recursive = true); - - /// Releases and removes the lock - /// Destructor - ~MutexImpl(); - - /// @} - /// @name Methods - /// @{ - public: - - /// Attempts to unconditionally acquire the lock. If the lock is held by - /// another thread, this method will wait until it can acquire the lock. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally acquire the lock. - bool acquire(); - - /// Attempts to release the lock. If the lock is held by the current - /// thread, the lock is released allowing other threads to acquire the - /// lock. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally release the lock. - bool release(); - - /// Attempts to acquire the lock without blocking. If the lock is not - /// available, this function returns false quickly (without blocking). If - /// the lock is available, it is acquired. - /// @returns false if any kind of error occurs or the lock is not - /// available, true otherwise. - /// Try to acquire the lock. - bool tryacquire(); - - //@} - /// @name Platform Dependent Data - /// @{ - private: -#if defined(LLVM_ENABLE_THREADS) && LLVM_ENABLE_THREADS != 0 - void* data_; ///< We don't know what the data will be -#endif - - /// @} - /// @name Do Not Implement - /// @{ - private: - MutexImpl(const MutexImpl &) = delete; - void operator=(const MutexImpl &) = delete; - /// @} - }; - - /// SmartMutex - A mutex with a compile time constant parameter that /// indicates whether this mutex should become a no-op when we're not /// running in multithreaded mode. template<bool mt_only> class SmartMutex { - MutexImpl impl; - unsigned acquired; - bool recursive; - public: - explicit SmartMutex(bool rec = true) : - impl(rec), acquired(0), recursive(rec) { } + std::recursive_mutex impl; + unsigned acquired = 0; + public: bool lock() { if (!mt_only || llvm_is_multithreaded()) { - return impl.acquire(); + impl.lock(); + return true; } else { // Single-threaded debugging code. This would be racy in // multithreaded mode, but provides not sanity checks in single // threaded mode. - assert((recursive || acquired == 0) && "Lock already acquired!!"); ++acquired; return true; } @@ -111,13 +45,13 @@ namespace llvm bool unlock() { if (!mt_only || llvm_is_multithreaded()) { - return impl.release(); + impl.unlock(); + return true; } else { // Single-threaded debugging code. This would be racy in // multithreaded mode, but provides not sanity checks in single // threaded mode. - assert(((recursive && acquired) || (acquired == 1)) && - "Lock not acquired before release!"); + assert(acquired && "Lock not acquired before release!"); --acquired; return true; } @@ -125,31 +59,16 @@ namespace llvm bool try_lock() { if (!mt_only || llvm_is_multithreaded()) - return impl.tryacquire(); + return impl.try_lock(); else return true; } - - private: - SmartMutex(const SmartMutex<mt_only> & original); - void operator=(const SmartMutex<mt_only> &); }; /// Mutex - A standard, always enforced mutex. typedef SmartMutex<false> Mutex; - template<bool mt_only> - class SmartScopedLock { - SmartMutex<mt_only>& mtx; - - public: - SmartScopedLock(SmartMutex<mt_only>& m) : mtx(m) { - mtx.lock(); - } - - ~SmartScopedLock() { - mtx.unlock(); - } - }; + template <bool mt_only> + using SmartScopedLock = std::lock_guard<SmartMutex<mt_only>>; typedef SmartScopedLock<false> ScopedLock; } diff --git a/include/llvm/Support/MutexGuard.h b/include/llvm/Support/MutexGuard.h deleted file mode 100644 index d86ced145816..000000000000 --- a/include/llvm/Support/MutexGuard.h +++ /dev/null @@ -1,40 +0,0 @@ -//===-- Support/MutexGuard.h - Acquire/Release Mutex In Scope ---*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines a guard for a block of code that ensures a Mutex is locked -// upon construction and released upon destruction. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_MUTEXGUARD_H -#define LLVM_SUPPORT_MUTEXGUARD_H - -#include "llvm/Support/Mutex.h" - -namespace llvm { - /// Instances of this class acquire a given Mutex Lock when constructed and - /// hold that lock until destruction. The intention is to instantiate one of - /// these on the stack at the top of some scope to be assured that C++ - /// destruction of the object will always release the Mutex and thus avoid - /// a host of nasty multi-threading problems in the face of exceptions, etc. - /// Guard a section of code with a Mutex. - class MutexGuard { - sys::Mutex &M; - MutexGuard(const MutexGuard &) = delete; - void operator=(const MutexGuard &) = delete; - public: - MutexGuard(sys::Mutex &m) : M(m) { M.lock(); } - ~MutexGuard() { M.unlock(); } - /// holds - Returns true if this locker instance holds the specified lock. - /// This is mostly used in assertions to validate that the correct mutex - /// is held. - bool holds(const sys::Mutex& lock) const { return &M == &lock; } - }; -} - -#endif // LLVM_SUPPORT_MUTEXGUARD_H diff --git a/include/llvm/Support/OnDiskHashTable.h b/include/llvm/Support/OnDiskHashTable.h index d84da92aab9b..11dc0de0f354 100644 --- a/include/llvm/Support/OnDiskHashTable.h +++ b/include/llvm/Support/OnDiskHashTable.h @@ -13,6 +13,7 @@ #ifndef LLVM_SUPPORT_ONDISKHASHTABLE_H #define LLVM_SUPPORT_ONDISKHASHTABLE_H +#include "llvm/Support/Alignment.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/EndianStream.h" @@ -207,7 +208,7 @@ public: // Pad with zeros so that we can start the hashtable at an aligned address. offset_type TableOff = Out.tell(); - uint64_t N = llvm::OffsetToAlignment(TableOff, alignof(offset_type)); + uint64_t N = offsetToAlignment(TableOff, Align(alignof(offset_type))); TableOff += N; while (N--) LE.write<uint8_t>(0); diff --git a/include/llvm/Support/Parallel.h b/include/llvm/Support/Parallel.h index eab9b492c4a5..3c0ed2c11127 100644 --- a/include/llvm/Support/Parallel.h +++ b/include/llvm/Support/Parallel.h @@ -18,14 +18,6 @@ #include <functional> #include <mutex> -#if defined(_MSC_VER) && LLVM_ENABLE_THREADS -#pragma warning(push) -#pragma warning(disable : 4530) -#include <concrt.h> -#include <ppl.h> -#pragma warning(pop) -#endif - namespace llvm { namespace parallel { @@ -84,23 +76,6 @@ public: void sync() const { L.sync(); } }; -#if defined(_MSC_VER) -template <class RandomAccessIterator, class Comparator> -void parallel_sort(RandomAccessIterator Start, RandomAccessIterator End, - const Comparator &Comp) { - concurrency::parallel_sort(Start, End, Comp); -} -template <class IterTy, class FuncTy> -void parallel_for_each(IterTy Begin, IterTy End, FuncTy Fn) { - concurrency::parallel_for_each(Begin, End, Fn); -} - -template <class IndexTy, class FuncTy> -void parallel_for_each_n(IndexTy Begin, IndexTy End, FuncTy Fn) { - concurrency::parallel_for(Begin, End, Fn); -} - -#else const ptrdiff_t MinParallelSize = 1024; /// Inclusive median. @@ -188,8 +163,6 @@ void parallel_for_each_n(IndexTy Begin, IndexTy End, FuncTy Fn) { #endif -#endif - template <typename Iter> using DefComparator = std::less<typename std::iterator_traits<Iter>::value_type>; diff --git a/include/llvm/Support/RWMutex.h b/include/llvm/Support/RWMutex.h index 9cd57cbd65a1..150bc7dbbce1 100644 --- a/include/llvm/Support/RWMutex.h +++ b/include/llvm/Support/RWMutex.h @@ -16,161 +16,184 @@ #include "llvm/Config/llvm-config.h" #include "llvm/Support/Threading.h" #include <cassert> +#include <mutex> +#include <shared_mutex> + +// std::shared_timed_mutex is only availble on macOS 10.12 and later. +#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) +#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101200 +#define LLVM_USE_RW_MUTEX_IMPL +#endif +#endif namespace llvm { namespace sys { - /// Platform agnostic RWMutex class. - class RWMutexImpl - { - /// @name Constructors - /// @{ - public: - - /// Initializes the lock but doesn't acquire it. - /// Default Constructor. - explicit RWMutexImpl(); - - /// @} - /// @name Do Not Implement - /// @{ - RWMutexImpl(const RWMutexImpl & original) = delete; - RWMutexImpl &operator=(const RWMutexImpl &) = delete; - /// @} - - /// Releases and removes the lock - /// Destructor - ~RWMutexImpl(); - - /// @} - /// @name Methods - /// @{ - public: - - /// Attempts to unconditionally acquire the lock in reader mode. If the - /// lock is held by a writer, this method will wait until it can acquire - /// the lock. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally acquire the lock in reader mode. - bool reader_acquire(); - - /// Attempts to release the lock in reader mode. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally release the lock in reader mode. - bool reader_release(); - - /// Attempts to unconditionally acquire the lock in reader mode. If the - /// lock is held by any readers, this method will wait until it can - /// acquire the lock. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally acquire the lock in writer mode. - bool writer_acquire(); - - /// Attempts to release the lock in writer mode. - /// @returns false if any kind of error occurs, true otherwise. - /// Unconditionally release the lock in write mode. - bool writer_release(); - - //@} - /// @name Platform Dependent Data - /// @{ - private: +#if defined(LLVM_USE_RW_MUTEX_IMPL) +/// Platform agnostic RWMutex class. +class RWMutexImpl { + /// @name Constructors + /// @{ +public: + /// Initializes the lock but doesn't acquire it. + /// Default Constructor. + explicit RWMutexImpl(); + + /// @} + /// @name Do Not Implement + /// @{ + RWMutexImpl(const RWMutexImpl &original) = delete; + RWMutexImpl &operator=(const RWMutexImpl &) = delete; + /// @} + + /// Releases and removes the lock + /// Destructor + ~RWMutexImpl(); + + /// @} + /// @name Methods + /// @{ +public: + /// Attempts to unconditionally acquire the lock in reader mode. If the + /// lock is held by a writer, this method will wait until it can acquire + /// the lock. + /// @returns false if any kind of error occurs, true otherwise. + /// Unconditionally acquire the lock in reader mode. + bool lock_shared(); + + /// Attempts to release the lock in reader mode. + /// @returns false if any kind of error occurs, true otherwise. + /// Unconditionally release the lock in reader mode. + bool unlock_shared(); + + /// Attempts to unconditionally acquire the lock in reader mode. If the + /// lock is held by any readers, this method will wait until it can + /// acquire the lock. + /// @returns false if any kind of error occurs, true otherwise. + /// Unconditionally acquire the lock in writer mode. + bool lock(); + + /// Attempts to release the lock in writer mode. + /// @returns false if any kind of error occurs, true otherwise. + /// Unconditionally release the lock in write mode. + bool unlock(); + + //@} + /// @name Platform Dependent Data + /// @{ +private: #if defined(LLVM_ENABLE_THREADS) && LLVM_ENABLE_THREADS != 0 - void* data_ = nullptr; ///< We don't know what the data will be + void *data_ = nullptr; ///< We don't know what the data will be +#endif +}; +#endif + +/// SmartMutex - An R/W mutex with a compile time constant parameter that +/// indicates whether this mutex should become a no-op when we're not +/// running in multithreaded mode. +template <bool mt_only> class SmartRWMutex { + // shared_mutex (C++17) is more efficient than shared_timed_mutex (C++14) + // on Windows and always available on MSVC. +#if defined(_MSC_VER) || __cplusplus > 201402L + std::shared_mutex impl; +#else +#if !defined(LLVM_USE_RW_MUTEX_IMPL) + std::shared_timed_mutex impl; +#else + RWMutexImpl impl; +#endif +#endif + unsigned readers = 0; + unsigned writers = 0; + +public: + bool lock_shared() { + if (!mt_only || llvm_is_multithreaded()) { + impl.lock_shared(); + return true; + } + + // Single-threaded debugging code. This would be racy in multithreaded + // mode, but provides not sanity checks in single threaded mode. + ++readers; + return true; + } + + bool unlock_shared() { + if (!mt_only || llvm_is_multithreaded()) { + impl.unlock_shared(); + return true; + } + + // Single-threaded debugging code. This would be racy in multithreaded + // mode, but provides not sanity checks in single threaded mode. + assert(readers > 0 && "Reader lock not acquired before release!"); + --readers; + return true; + } + + bool lock() { + if (!mt_only || llvm_is_multithreaded()) { + impl.lock(); + return true; + } + + // Single-threaded debugging code. This would be racy in multithreaded + // mode, but provides not sanity checks in single threaded mode. + assert(writers == 0 && "Writer lock already acquired!"); + ++writers; + return true; + } + + bool unlock() { + if (!mt_only || llvm_is_multithreaded()) { + impl.unlock(); + return true; + } + + // Single-threaded debugging code. This would be racy in multithreaded + // mode, but provides not sanity checks in single threaded mode. + assert(writers == 1 && "Writer lock not acquired before release!"); + --writers; + return true; + } +}; + +typedef SmartRWMutex<false> RWMutex; + +/// ScopedReader - RAII acquisition of a reader lock +#if !defined(LLVM_USE_RW_MUTEX_IMPL) +template <bool mt_only> +using SmartScopedReader = const std::shared_lock<SmartRWMutex<mt_only>>; +#else +template <bool mt_only> struct SmartScopedReader { + SmartRWMutex<mt_only> &mutex; + + explicit SmartScopedReader(SmartRWMutex<mt_only> &m) : mutex(m) { + mutex.lock_shared(); + } + + ~SmartScopedReader() { mutex.unlock_shared(); } +}; +#endif +typedef SmartScopedReader<false> ScopedReader; + +/// ScopedWriter - RAII acquisition of a writer lock +#if !defined(LLVM_USE_RW_MUTEX_IMPL) +template <bool mt_only> +using SmartScopedWriter = std::lock_guard<SmartRWMutex<mt_only>>; +#else +template <bool mt_only> struct SmartScopedWriter { + SmartRWMutex<mt_only> &mutex; + + explicit SmartScopedWriter(SmartRWMutex<mt_only> &m) : mutex(m) { + mutex.lock(); + } + + ~SmartScopedWriter() { mutex.unlock(); } +}; #endif - }; - - /// SmartMutex - An R/W mutex with a compile time constant parameter that - /// indicates whether this mutex should become a no-op when we're not - /// running in multithreaded mode. - template<bool mt_only> - class SmartRWMutex { - RWMutexImpl impl; - unsigned readers = 0; - unsigned writers = 0; - - public: - explicit SmartRWMutex() = default; - SmartRWMutex(const SmartRWMutex<mt_only> & original) = delete; - SmartRWMutex<mt_only> &operator=(const SmartRWMutex<mt_only> &) = delete; - - bool lock_shared() { - if (!mt_only || llvm_is_multithreaded()) - return impl.reader_acquire(); - - // Single-threaded debugging code. This would be racy in multithreaded - // mode, but provides not sanity checks in single threaded mode. - ++readers; - return true; - } - - bool unlock_shared() { - if (!mt_only || llvm_is_multithreaded()) - return impl.reader_release(); - - // Single-threaded debugging code. This would be racy in multithreaded - // mode, but provides not sanity checks in single threaded mode. - assert(readers > 0 && "Reader lock not acquired before release!"); - --readers; - return true; - } - - bool lock() { - if (!mt_only || llvm_is_multithreaded()) - return impl.writer_acquire(); - - // Single-threaded debugging code. This would be racy in multithreaded - // mode, but provides not sanity checks in single threaded mode. - assert(writers == 0 && "Writer lock already acquired!"); - ++writers; - return true; - } - - bool unlock() { - if (!mt_only || llvm_is_multithreaded()) - return impl.writer_release(); - - // Single-threaded debugging code. This would be racy in multithreaded - // mode, but provides not sanity checks in single threaded mode. - assert(writers == 1 && "Writer lock not acquired before release!"); - --writers; - return true; - } - }; - - typedef SmartRWMutex<false> RWMutex; - - /// ScopedReader - RAII acquisition of a reader lock - template<bool mt_only> - struct SmartScopedReader { - SmartRWMutex<mt_only>& mutex; - - explicit SmartScopedReader(SmartRWMutex<mt_only>& m) : mutex(m) { - mutex.lock_shared(); - } - - ~SmartScopedReader() { - mutex.unlock_shared(); - } - }; - - typedef SmartScopedReader<false> ScopedReader; - - /// ScopedWriter - RAII acquisition of a writer lock - template<bool mt_only> - struct SmartScopedWriter { - SmartRWMutex<mt_only>& mutex; - - explicit SmartScopedWriter(SmartRWMutex<mt_only>& m) : mutex(m) { - mutex.lock(); - } - - ~SmartScopedWriter() { - mutex.unlock(); - } - }; - - typedef SmartScopedWriter<false> ScopedWriter; +typedef SmartScopedWriter<false> ScopedWriter; } // end namespace sys } // end namespace llvm diff --git a/include/llvm/Support/Regex.h b/include/llvm/Support/Regex.h index 2d19b10fd890..b2620ab4cfc9 100644 --- a/include/llvm/Support/Regex.h +++ b/include/llvm/Support/Regex.h @@ -44,6 +44,9 @@ namespace llvm { Regex(); /// Compiles the given regular expression \p Regex. + /// + /// \param Regex - referenced string is no longer needed after this + /// constructor does finish. Only its compiled form is kept stored. Regex(StringRef Regex, unsigned Flags = NoFlags); Regex(const Regex &) = delete; Regex &operator=(Regex regex) { @@ -54,9 +57,10 @@ namespace llvm { Regex(Regex &®ex); ~Regex(); - /// isValid - returns the error encountered during regex compilation, or - /// matching, if any. + /// isValid - returns the error encountered during regex compilation, if + /// any. bool isValid(std::string &Error) const; + bool isValid() const { return !error; } /// getNumMatches - In a valid regex, return the number of parenthesized /// matches it contains. The number filled in by match will include this @@ -69,8 +73,12 @@ namespace llvm { /// with references to the matched group expressions (inside \p String), /// the first group is always the entire pattern. /// + /// \param Error - If non-null, any errors in the matching will be recorded + /// as a non-empty string. If there is no error, it will be an empty string. + /// /// This returns true on a successful match. - bool match(StringRef String, SmallVectorImpl<StringRef> *Matches = nullptr); + bool match(StringRef String, SmallVectorImpl<StringRef> *Matches = nullptr, + std::string *Error = nullptr) const; /// sub - Return the result of replacing the first match of the regex in /// \p String with the \p Repl string. Backreferences like "\0" in the @@ -81,9 +89,9 @@ namespace llvm { /// /// \param Error If non-null, any errors in the substitution (invalid /// backreferences, trailing backslashes) will be recorded as a non-empty - /// string. + /// string. If there is no error, it will be an empty string. std::string sub(StringRef Repl, StringRef String, - std::string *Error = nullptr); + std::string *Error = nullptr) const; /// If this function returns true, ^Str$ is an extended regular /// expression that matches Str and only Str. diff --git a/include/llvm/Support/Registry.h b/include/llvm/Support/Registry.h index 4d8aa5f1470d..5bb6a254a47f 100644 --- a/include/llvm/Support/Registry.h +++ b/include/llvm/Support/Registry.h @@ -115,7 +115,7 @@ namespace llvm { entry Entry; node Node; - static std::unique_ptr<T> CtorFn() { return make_unique<V>(); } + static std::unique_ptr<T> CtorFn() { return std::make_unique<V>(); } public: Add(StringRef Name, StringRef Desc) diff --git a/include/llvm/Support/SHA1.h b/include/llvm/Support/SHA1.h index 87fe94bbd5cd..2cfbd2179364 100644 --- a/include/llvm/Support/SHA1.h +++ b/include/llvm/Support/SHA1.h @@ -16,13 +16,13 @@ #define LLVM_SUPPORT_SHA1_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" #include <array> #include <cstdint> namespace llvm { template <typename T> class ArrayRef; -class StringRef; /// A class that wrap the SHA1 algorithm. class SHA1 { diff --git a/include/llvm/Support/ScalableSize.h b/include/llvm/Support/ScalableSize.h deleted file mode 100644 index 96bf043773a0..000000000000 --- a/include/llvm/Support/ScalableSize.h +++ /dev/null @@ -1,43 +0,0 @@ -//===- ScalableSize.h - Scalable vector size info ---------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file provides a struct that can be used to query the size of IR types -// which may be scalable vectors. It provides convenience operators so that -// it can be used in much the same way as a single scalar value. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_SCALABLESIZE_H -#define LLVM_SUPPORT_SCALABLESIZE_H - -namespace llvm { - -class ElementCount { -public: - unsigned Min; // Minimum number of vector elements. - bool Scalable; // If true, NumElements is a multiple of 'Min' determined - // at runtime rather than compile time. - - ElementCount(unsigned Min, bool Scalable) - : Min(Min), Scalable(Scalable) {} - - ElementCount operator*(unsigned RHS) { - return { Min * RHS, Scalable }; - } - ElementCount operator/(unsigned RHS) { - return { Min / RHS, Scalable }; - } - - bool operator==(const ElementCount& RHS) const { - return Min == RHS.Min && Scalable == RHS.Scalable; - } -}; - -} // end namespace llvm - -#endif // LLVM_SUPPORT_SCALABLESIZE_H diff --git a/include/llvm/Support/Signals.h b/include/llvm/Support/Signals.h index a6b215a24311..a4f1fad22dd5 100644 --- a/include/llvm/Support/Signals.h +++ b/include/llvm/Support/Signals.h @@ -84,6 +84,17 @@ namespace sys { /// function. Note also that the handler may be executed on a different /// thread on some platforms. void SetInfoSignalFunction(void (*Handler)()); + + /// Registers a function to be called when a "pipe" signal is delivered to + /// the process. + /// + /// The "pipe" signal typically indicates a failed write to a pipe (SIGPIPE). + /// The default installed handler calls `exit(EX_IOERR)`, causing the process + /// to immediately exit with an IO error exit code. + /// + /// This function is only applicable on POSIX systems. + void SetPipeSignalFunction(void (*Handler)()); + } // End sys namespace } // End llvm namespace diff --git a/include/llvm/Support/SwapByteOrder.h b/include/llvm/Support/SwapByteOrder.h index 06a447a27c2a..6cec87006c02 100644 --- a/include/llvm/Support/SwapByteOrder.h +++ b/include/llvm/Support/SwapByteOrder.h @@ -22,9 +22,37 @@ #include <stdlib.h> #endif +#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) +#include <endian.h> +#elif defined(_AIX) +#include <sys/machine.h> +#elif defined(__sun) +/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */ +#include <sys/types.h> +#define BIG_ENDIAN 4321 +#define LITTLE_ENDIAN 1234 +#if defined(_BIG_ENDIAN) +#define BYTE_ORDER BIG_ENDIAN +#else +#define BYTE_ORDER LITTLE_ENDIAN +#endif +#else +#if !defined(BYTE_ORDER) && !defined(_WIN32) +#include <machine/endian.h> +#endif +#endif + namespace llvm { namespace sys { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN +constexpr bool IsBigEndianHost = true; +#else +constexpr bool IsBigEndianHost = false; +#endif + +static const bool IsLittleEndianHost = !IsBigEndianHost; + /// SwapByteOrder_16 - This function returns a byte-swapped representation of /// the 16-bit argument. inline uint16_t SwapByteOrder_16(uint16_t value) { @@ -39,10 +67,9 @@ inline uint16_t SwapByteOrder_16(uint16_t value) { #endif } -/// SwapByteOrder_32 - This function returns a byte-swapped representation of -/// the 32-bit argument. +/// This function returns a byte-swapped representation of the 32-bit argument. inline uint32_t SwapByteOrder_32(uint32_t value) { -#if defined(__llvm__) || (LLVM_GNUC_PREREQ(4, 3, 0) && !defined(__ICC)) +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) return __builtin_bswap32(value); #elif defined(_MSC_VER) && !defined(_DEBUG) return _byteswap_ulong(value); @@ -55,10 +82,9 @@ inline uint32_t SwapByteOrder_32(uint32_t value) { #endif } -/// SwapByteOrder_64 - This function returns a byte-swapped representation of -/// the 64-bit argument. +/// This function returns a byte-swapped representation of the 64-bit argument. inline uint64_t SwapByteOrder_64(uint64_t value) { -#if defined(__llvm__) || (LLVM_GNUC_PREREQ(4, 3, 0) && !defined(__ICC)) +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) return __builtin_bswap64(value); #elif defined(_MSC_VER) && !defined(_DEBUG) return _byteswap_uint64(value); diff --git a/include/llvm/Support/TargetOpcodes.def b/include/llvm/Support/TargetOpcodes.def index 598c1064efd0..11731ac35415 100644 --- a/include/llvm/Support/TargetOpcodes.def +++ b/include/llvm/Support/TargetOpcodes.def @@ -294,9 +294,21 @@ HANDLE_TARGET_OPCODE(G_SEXTLOAD) /// Generic zeroext load HANDLE_TARGET_OPCODE(G_ZEXTLOAD) +/// Generic indexed load (including anyext load) +HANDLE_TARGET_OPCODE(G_INDEXED_LOAD) + +/// Generic indexed signext load +HANDLE_TARGET_OPCODE(G_INDEXED_SEXTLOAD) + +/// Generic indexed zeroext load +HANDLE_TARGET_OPCODE(G_INDEXED_ZEXTLOAD) + /// Generic store. HANDLE_TARGET_OPCODE(G_STORE) +/// Generic indexed store. +HANDLE_TARGET_OPCODE(G_INDEXED_STORE) + /// Generic atomic cmpxchg with internal success check. HANDLE_TARGET_OPCODE(G_ATOMIC_CMPXCHG_WITH_SUCCESS) @@ -315,6 +327,8 @@ HANDLE_TARGET_OPCODE(G_ATOMICRMW_MAX) HANDLE_TARGET_OPCODE(G_ATOMICRMW_MIN) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMAX) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMIN) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_FADD) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_FSUB) // Generic atomic fence HANDLE_TARGET_OPCODE(G_FENCE) @@ -354,6 +368,7 @@ HANDLE_TARGET_OPCODE(G_VAARG) // Generic sign extend HANDLE_TARGET_OPCODE(G_SEXT) +HANDLE_TARGET_OPCODE(G_SEXT_INREG) // Generic zero extend HANDLE_TARGET_OPCODE(G_ZEXT) @@ -436,6 +451,9 @@ HANDLE_TARGET_OPCODE(G_FMUL) /// Generic FMA multiplication. Behaves like llvm fma intrinsic HANDLE_TARGET_OPCODE(G_FMA) +/// Generic FP multiply and add. Behaves as separate fmul and fadd. +HANDLE_TARGET_OPCODE(G_FMAD) + /// Generic FP division. HANDLE_TARGET_OPCODE(G_FDIV) @@ -557,6 +575,9 @@ HANDLE_TARGET_OPCODE(G_CTPOP) /// Generic byte swap. HANDLE_TARGET_OPCODE(G_BSWAP) +/// Generic bit reverse. +HANDLE_TARGET_OPCODE(G_BITREVERSE) + /// Floating point ceil. HANDLE_TARGET_OPCODE(G_FCEIL) @@ -587,12 +608,15 @@ HANDLE_TARGET_OPCODE(G_BLOCK_ADDR) /// Generic jump table address HANDLE_TARGET_OPCODE(G_JUMP_TABLE) +/// Generic dynamic stack allocation. +HANDLE_TARGET_OPCODE(G_DYN_STACKALLOC) + // TODO: Add more generic opcodes as we move along. /// Marker for the end of the generic opcode. /// This is used to check if an opcode is in the range of the /// generic opcodes. -HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_JUMP_TABLE) +HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_DYN_STACKALLOC) /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific post-isel opcode values start here. diff --git a/include/llvm/Support/TargetRegistry.h b/include/llvm/Support/TargetRegistry.h index bf75650760d0..f4bc26b858c8 100644 --- a/include/llvm/Support/TargetRegistry.h +++ b/include/llvm/Support/TargetRegistry.h @@ -510,8 +510,8 @@ public: std::move(Emitter), RelaxAll); break; case Triple::XCOFF: - S = createXCOFFStreamer(Ctx, std::move(TAB), std::move(OW), - std::move(Emitter), RelaxAll); + S = createXCOFFStreamer(Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); break; } if (ObjectTargetStreamerCtorFn) diff --git a/include/llvm/Support/TimeProfiler.h b/include/llvm/Support/TimeProfiler.h index 72b6f7180bde..8cc430d0bc72 100644 --- a/include/llvm/Support/TimeProfiler.h +++ b/include/llvm/Support/TimeProfiler.h @@ -19,7 +19,7 @@ extern TimeTraceProfiler *TimeTraceProfilerInstance; /// Initialize the time trace profiler. /// This sets up the global \p TimeTraceProfilerInstance /// variable to be the profiler instance. -void timeTraceProfilerInitialize(); +void timeTraceProfilerInitialize(unsigned TimeTraceGranularity); /// Cleanup the time trace profiler, if it was initialized. void timeTraceProfilerCleanup(); diff --git a/include/llvm/Support/TrailingObjects.h b/include/llvm/Support/TrailingObjects.h index 8cf4f7aed7f8..49be89613c43 100644 --- a/include/llvm/Support/TrailingObjects.h +++ b/include/llvm/Support/TrailingObjects.h @@ -47,6 +47,7 @@ #define LLVM_SUPPORT_TRAILINGOBJECTS_H #include "llvm/Support/AlignOf.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/type_traits.h" @@ -87,11 +88,6 @@ protected: template <typename T> struct OverloadToken {}; }; -/// This helper template works-around MSVC 2013's lack of useful -/// alignas() support. The argument to alignas(), in MSVC, is -/// required to be a literal integer. But, you *can* use template -/// specialization to select between a bunch of different alignas() -/// expressions... template <int Align> class TrailingObjectsAligner : public TrailingObjectsBase {}; template <> @@ -172,7 +168,7 @@ protected: if (requiresRealignment()) return reinterpret_cast<const NextTy *>( - llvm::alignAddr(Ptr, alignof(NextTy))); + alignAddr(Ptr, Align::Of<NextTy>())); else return reinterpret_cast<const NextTy *>(Ptr); } @@ -186,7 +182,7 @@ protected: Obj, TrailingObjectsBase::OverloadToken<PrevTy>()); if (requiresRealignment()) - return reinterpret_cast<NextTy *>(llvm::alignAddr(Ptr, alignof(NextTy))); + return reinterpret_cast<NextTy *>(alignAddr(Ptr, Align::Of<NextTy>())); else return reinterpret_cast<NextTy *>(Ptr); } @@ -254,9 +250,7 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl< // because BaseTy isn't complete at class instantiation time, but // will be by the time this function is instantiated. static void verifyTrailingObjectsAssertions() { -#ifdef LLVM_IS_FINAL - static_assert(LLVM_IS_FINAL(BaseTy), "BaseTy must be final."); -#endif + static_assert(std::is_final<BaseTy>(), "BaseTy must be final."); } // These two methods are the base of the recursion for this method. @@ -369,7 +363,9 @@ public: template <typename... Tys> struct FixedSizeStorage { template <size_t... Counts> struct with_counts { enum { Size = totalSizeToAlloc<Tys...>(Counts...) }; - typedef llvm::AlignedCharArray<alignof(BaseTy), Size> type; + struct type { + alignas(BaseTy) char buffer[Size]; + }; }; }; diff --git a/include/llvm/Support/TypeSize.h b/include/llvm/Support/TypeSize.h new file mode 100644 index 000000000000..711679cdcacb --- /dev/null +++ b/include/llvm/Support/TypeSize.h @@ -0,0 +1,201 @@ +//===- TypeSize.h - Wrapper around type sizes -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a struct that can be used to query the size of IR types +// which may be scalable vectors. It provides convenience operators so that +// it can be used in much the same way as a single scalar value. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_TYPESIZE_H +#define LLVM_SUPPORT_TYPESIZE_H + +#include <cassert> +#include <tuple> + +namespace llvm { + +class ElementCount { +public: + unsigned Min; // Minimum number of vector elements. + bool Scalable; // If true, NumElements is a multiple of 'Min' determined + // at runtime rather than compile time. + + ElementCount(unsigned Min, bool Scalable) + : Min(Min), Scalable(Scalable) {} + + ElementCount operator*(unsigned RHS) { + return { Min * RHS, Scalable }; + } + ElementCount operator/(unsigned RHS) { + return { Min / RHS, Scalable }; + } + + bool operator==(const ElementCount& RHS) const { + return Min == RHS.Min && Scalable == RHS.Scalable; + } + bool operator!=(const ElementCount& RHS) const { + return !(*this == RHS); + } +}; + +// This class is used to represent the size of types. If the type is of fixed +// size, it will represent the exact size. If the type is a scalable vector, +// it will represent the known minimum size. +class TypeSize { + uint64_t MinSize; // The known minimum size. + bool IsScalable; // If true, then the runtime size is an integer multiple + // of MinSize. + +public: + constexpr TypeSize(uint64_t MinSize, bool Scalable) + : MinSize(MinSize), IsScalable(Scalable) {} + + static constexpr TypeSize Fixed(uint64_t Size) { + return TypeSize(Size, /*IsScalable=*/false); + } + + static constexpr TypeSize Scalable(uint64_t MinSize) { + return TypeSize(MinSize, /*IsScalable=*/true); + } + + // Scalable vector types with the same minimum size as a fixed size type are + // not guaranteed to be the same size at runtime, so they are never + // considered to be equal. + friend bool operator==(const TypeSize &LHS, const TypeSize &RHS) { + return std::tie(LHS.MinSize, LHS.IsScalable) == + std::tie(RHS.MinSize, RHS.IsScalable); + } + + friend bool operator!=(const TypeSize &LHS, const TypeSize &RHS) { + return !(LHS == RHS); + } + + // For many cases, size ordering between scalable and fixed size types cannot + // be determined at compile time, so such comparisons aren't allowed. + // + // e.g. <vscale x 2 x i16> could be bigger than <4 x i32> with a runtime + // vscale >= 5, equal sized with a vscale of 4, and smaller with + // a vscale <= 3. + // + // If the scalable flags match, just perform the requested comparison + // between the minimum sizes. + friend bool operator<(const TypeSize &LHS, const TypeSize &RHS) { + assert(LHS.IsScalable == RHS.IsScalable && + "Ordering comparison of scalable and fixed types"); + + return LHS.MinSize < RHS.MinSize; + } + + friend bool operator>(const TypeSize &LHS, const TypeSize &RHS) { + return RHS < LHS; + } + + friend bool operator<=(const TypeSize &LHS, const TypeSize &RHS) { + return !(RHS < LHS); + } + + friend bool operator>=(const TypeSize &LHS, const TypeSize& RHS) { + return !(LHS < RHS); + } + + // Convenience operators to obtain relative sizes independently of + // the scalable flag. + TypeSize operator*(unsigned RHS) const { + return { MinSize * RHS, IsScalable }; + } + + friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) { + return { LHS * RHS.MinSize, RHS.IsScalable }; + } + + TypeSize operator/(unsigned RHS) const { + return { MinSize / RHS, IsScalable }; + } + + // Return the minimum size with the assumption that the size is exact. + // Use in places where a scalable size doesn't make sense (e.g. non-vector + // types, or vectors in backends which don't support scalable vectors). + uint64_t getFixedSize() const { + assert(!IsScalable && "Request for a fixed size on a scalable object"); + return MinSize; + } + + // Return the known minimum size. Use in places where the scalable property + // doesn't matter (e.g. determining alignment) or in conjunction with the + // isScalable method below. + uint64_t getKnownMinSize() const { + return MinSize; + } + + // Return whether or not the size is scalable. + bool isScalable() const { + return IsScalable; + } + + // Casts to a uint64_t if this is a fixed-width size. + // + // NOTE: This interface is obsolete and will be removed in a future version + // of LLVM in favour of calling getFixedSize() directly. + operator uint64_t() const { + return getFixedSize(); + } + + // Additional convenience operators needed to avoid ambiguous parses. + // TODO: Make uint64_t the default operator? + TypeSize operator*(uint64_t RHS) const { + return { MinSize * RHS, IsScalable }; + } + + TypeSize operator*(int RHS) const { + return { MinSize * RHS, IsScalable }; + } + + TypeSize operator*(int64_t RHS) const { + return { MinSize * RHS, IsScalable }; + } + + friend TypeSize operator*(const uint64_t LHS, const TypeSize &RHS) { + return { LHS * RHS.MinSize, RHS.IsScalable }; + } + + friend TypeSize operator*(const int LHS, const TypeSize &RHS) { + return { LHS * RHS.MinSize, RHS.IsScalable }; + } + + friend TypeSize operator*(const int64_t LHS, const TypeSize &RHS) { + return { LHS * RHS.MinSize, RHS.IsScalable }; + } + + TypeSize operator/(uint64_t RHS) const { + return { MinSize / RHS, IsScalable }; + } + + TypeSize operator/(int RHS) const { + return { MinSize / RHS, IsScalable }; + } + + TypeSize operator/(int64_t RHS) const { + return { MinSize / RHS, IsScalable }; + } +}; + +/// Returns a TypeSize with a known minimum size that is the next integer +/// (mod 2**64) that is greater than or equal to \p Value and is a multiple +/// of \p Align. \p Align must be non-zero. +/// +/// Similar to the alignTo functions in MathExtras.h +inline TypeSize alignTo(TypeSize Size, uint64_t Align) { + assert(Align != 0u && "Align must be non-zero"); + return {(Size.getKnownMinSize() + Align - 1) / Align * Align, + Size.isScalable()}; +} + +} // end namespace llvm + +#endif // LLVM_SUPPORT_TypeSize_H diff --git a/include/llvm/Support/UnicodeCharRanges.h b/include/llvm/Support/UnicodeCharRanges.h index 4b59f8a92b76..73d3603b74df 100644 --- a/include/llvm/Support/UnicodeCharRanges.h +++ b/include/llvm/Support/UnicodeCharRanges.h @@ -9,11 +9,8 @@ #define LLVM_SUPPORT_UNICODECHARRANGES_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/Mutex.h" -#include "llvm/Support/MutexGuard.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> diff --git a/include/llvm/Support/UniqueLock.h b/include/llvm/Support/UniqueLock.h deleted file mode 100644 index 0a887ad5965d..000000000000 --- a/include/llvm/Support/UniqueLock.h +++ /dev/null @@ -1,68 +0,0 @@ -//===- Support/UniqueLock.h - Acquire/Release Mutex In Scope ----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines a guard for a block of code that ensures a Mutex is locked -// upon construction and released upon destruction. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_UNIQUE_LOCK_H -#define LLVM_SUPPORT_UNIQUE_LOCK_H - -#include <cassert> - -namespace llvm { - - /// A pared-down imitation of std::unique_lock from C++11. Contrary to the - /// name, it's really more of a wrapper for a lock. It may or may not have - /// an associated mutex, which is guaranteed to be locked upon creation - /// and unlocked after destruction. unique_lock can also unlock the mutex - /// and re-lock it freely during its lifetime. - /// Guard a section of code with a mutex. - template<typename MutexT> - class unique_lock { - MutexT *M = nullptr; - bool locked = false; - - public: - unique_lock() = default; - explicit unique_lock(MutexT &m) : M(&m), locked(true) { M->lock(); } - unique_lock(const unique_lock &) = delete; - unique_lock &operator=(const unique_lock &) = delete; - - void operator=(unique_lock &&o) { - if (owns_lock()) - M->unlock(); - M = o.M; - locked = o.locked; - o.M = nullptr; - o.locked = false; - } - - ~unique_lock() { if (owns_lock()) M->unlock(); } - - void lock() { - assert(!locked && "mutex already locked!"); - assert(M && "no associated mutex!"); - M->lock(); - locked = true; - } - - void unlock() { - assert(locked && "unlocking a mutex that isn't locked!"); - assert(M && "no associated mutex!"); - M->unlock(); - locked = false; - } - - bool owns_lock() { return locked; } - }; - -} // end namespace llvm - -#endif // LLVM_SUPPORT_UNIQUE_LOCK_H diff --git a/include/llvm/Support/VirtualFileSystem.h b/include/llvm/Support/VirtualFileSystem.h index 31c9e851daed..c844d9d194f0 100644 --- a/include/llvm/Support/VirtualFileSystem.h +++ b/include/llvm/Support/VirtualFileSystem.h @@ -647,9 +647,19 @@ private: friend class VFSFromYamlDirIterImpl; friend class RedirectingFileSystemParser; + bool shouldUseExternalFS() const { + return ExternalFSValidWD && IsFallthrough; + } + /// The root(s) of the virtual file system. std::vector<std::unique_ptr<Entry>> Roots; + /// The current working directory of the file system. + std::string WorkingDirectory; + + /// Whether the current working directory is valid for the external FS. + bool ExternalFSValidWD = false; + /// The file system to use for external references. IntrusiveRefCntPtr<FileSystem> ExternalFS; @@ -689,8 +699,7 @@ private: true; #endif - RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS) - : ExternalFS(std::move(ExternalFS)) {} + RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS); /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly /// recursing into the contents of \p From if it is a directory. @@ -730,9 +739,10 @@ public: StringRef getExternalContentsPrefixDir() const; + void dump(raw_ostream &OS) const; + void dumpEntry(raw_ostream &OS, Entry *E, int NumSpaces = 0) const; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void dump() const; - LLVM_DUMP_METHOD void dumpEntry(Entry *E, int NumSpaces = 0) const; #endif }; diff --git a/include/llvm/Support/Win64EH.h b/include/llvm/Support/Win64EH.h index bdd23b41594e..8220131e5be9 100644 --- a/include/llvm/Support/Win64EH.h +++ b/include/llvm/Support/Win64EH.h @@ -30,7 +30,9 @@ enum UnwindOpcodes { UOP_SetFPReg, UOP_SaveNonVol, UOP_SaveNonVolBig, - UOP_SaveXMM128 = 8, + UOP_Epilog, + UOP_SpareCode, + UOP_SaveXMM128, UOP_SaveXMM128Big, UOP_PushMachFrame, // The following set of unwind opcodes is for ARM64. They are documented at diff --git a/include/llvm/Support/X86TargetParser.def b/include/llvm/Support/X86TargetParser.def index 1749be3b3ae2..4ebf2d79cb8d 100644 --- a/include/llvm/Support/X86TargetParser.def +++ b/include/llvm/Support/X86TargetParser.def @@ -112,6 +112,7 @@ X86_CPU_SUBTYPE ("k6-2", AMDPENTIUM_K62) X86_CPU_SUBTYPE ("k6-3", AMDPENTIUM_K63) X86_CPU_SUBTYPE ("geode", AMDPENTIUM_GEODE) X86_CPU_SUBTYPE ("cooperlake", INTEL_COREI7_COOPERLAKE) +X86_CPU_SUBTYPE ("tigerlake", INTEL_COREI7_TIGERLAKE) #undef X86_CPU_SUBTYPE_COMPAT #undef X86_CPU_SUBTYPE @@ -160,12 +161,13 @@ X86_FEATURE_COMPAT(32, FEATURE_GFNI, "gfni") X86_FEATURE_COMPAT(33, FEATURE_VPCLMULQDQ, "vpclmulqdq") X86_FEATURE_COMPAT(34, FEATURE_AVX512VNNI, "avx512vnni") X86_FEATURE_COMPAT(35, FEATURE_AVX512BITALG, "avx512bitalg") +X86_FEATURE_COMPAT(36, FEATURE_AVX512BF16, "avx512bf16") // Features below here are not in libgcc/compiler-rt. X86_FEATURE (64, FEATURE_MOVBE) X86_FEATURE (65, FEATURE_ADX) X86_FEATURE (66, FEATURE_EM64T) X86_FEATURE (67, FEATURE_CLFLUSHOPT) X86_FEATURE (68, FEATURE_SHA) -X86_FEATURE (69, FEATURE_AVX512BF16) +X86_FEATURE (69, FEATURE_AVX512VP2INTERSECT) #undef X86_FEATURE_COMPAT #undef X86_FEATURE diff --git a/include/llvm/Support/YAMLTraits.h b/include/llvm/Support/YAMLTraits.h index 5181dc56d81d..a3bfa7dc4678 100644 --- a/include/llvm/Support/YAMLTraits.h +++ b/include/llvm/Support/YAMLTraits.h @@ -649,7 +649,8 @@ inline bool isBool(StringRef S) { inline QuotingType needsQuotes(StringRef S) { if (S.empty()) return QuotingType::Single; - if (isspace(S.front()) || isspace(S.back())) + if (isspace(static_cast<unsigned char>(S.front())) || + isspace(static_cast<unsigned char>(S.back()))) return QuotingType::Single; if (isNull(S)) return QuotingType::Single; @@ -748,7 +749,7 @@ public: IO(void *Ctxt = nullptr); virtual ~IO(); - virtual bool outputting() = 0; + virtual bool outputting() const = 0; virtual unsigned beginSequence() = 0; virtual bool preflightElement(unsigned, void *&) = 0; @@ -842,7 +843,7 @@ public: Val = Val | ConstVal; } - void *getContext(); + void *getContext() const; void setContext(void *); template <typename T> void mapRequired(const char *Key, T &Val) { @@ -1402,7 +1403,7 @@ public: std::error_code error(); private: - bool outputting() override; + bool outputting() const override; bool mapTag(StringRef, bool) override; void beginMapping() override; void endMapping() override; @@ -1549,7 +1550,7 @@ public: /// anyway. void setWriteDefaultValues(bool Write) { WriteDefaultValues = Write; } - bool outputting() override; + bool outputting() const override; bool mapTag(StringRef, bool) override; void beginMapping() override; void endMapping() override; diff --git a/include/llvm/Support/circular_raw_ostream.h b/include/llvm/Support/circular_raw_ostream.h index 4ecdb17376f1..a72acd4fe002 100644 --- a/include/llvm/Support/circular_raw_ostream.h +++ b/include/llvm/Support/circular_raw_ostream.h @@ -122,6 +122,10 @@ namespace llvm { delete[] BufferArray; } + bool is_displayed() const override { + return TheStream->is_displayed(); + } + /// setStream - Tell the circular_raw_ostream to output a /// different stream. "Owns" tells circular_raw_ostream whether /// it should take responsibility for managing the underlying diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h index 48bb623b0638..0debc5da7a68 100644 --- a/include/llvm/Support/raw_ostream.h +++ b/include/llvm/Support/raw_ostream.h @@ -72,7 +72,7 @@ private: public: // color order matches ANSI escape sequence, don't change - enum Colors { + enum class Colors { BLACK = 0, RED, GREEN, @@ -81,9 +81,21 @@ public: MAGENTA, CYAN, WHITE, - SAVEDCOLOR + SAVEDCOLOR, + RESET, }; + static const Colors BLACK = Colors::BLACK; + static const Colors RED = Colors::RED; + static const Colors GREEN = Colors::GREEN; + static const Colors YELLOW = Colors::YELLOW; + static const Colors BLUE = Colors::BLUE; + static const Colors MAGENTA = Colors::MAGENTA; + static const Colors CYAN = Colors::CYAN; + static const Colors WHITE = Colors::WHITE; + static const Colors SAVEDCOLOR = Colors::SAVEDCOLOR; + static const Colors RESET = Colors::RESET; + explicit raw_ostream(bool unbuffered = false) : BufferMode(unbuffered ? Unbuffered : InternalBuffer) { // Start out ready to flush. @@ -214,6 +226,9 @@ public: /// Output \p N in hexadecimal, without any prefix or padding. raw_ostream &write_hex(unsigned long long N); + // Change the foreground color of text. + raw_ostream &operator<<(Colors C); + /// Output a formatted UUID with dash separators. using uuid_t = uint8_t[16]; raw_ostream &write_uuid(const uuid_t UUID); @@ -277,6 +292,10 @@ public: /// This function determines if this stream is displayed and supports colors. virtual bool has_colors() const { return is_displayed(); } + // Enable or disable colors. Once disable_colors() is called, + // changeColor() has no effect until enable_colors() is called. + virtual void enable_colors(bool /*enable*/) {} + //===--------------------------------------------------------------------===// // Subclass Interface //===--------------------------------------------------------------------===// @@ -365,8 +384,8 @@ public: class raw_fd_ostream : public raw_pwrite_stream { int FD; bool ShouldClose; - bool SupportsSeeking; + bool ColorEnabled = true; #ifdef _WIN32 /// True if this fd refers to a Windows console device. Mintty and other @@ -442,6 +461,8 @@ public: bool has_colors() const override; + void enable_colors(bool enable) override { ColorEnabled = enable; } + std::error_code error() const { return EC; } /// Return the value of the flag in this raw_fd_ostream indicating whether an diff --git a/include/llvm/Support/type_traits.h b/include/llvm/Support/type_traits.h index c8c6a76a90f1..b7d48e8e1ade 100644 --- a/include/llvm/Support/type_traits.h +++ b/include/llvm/Support/type_traits.h @@ -17,11 +17,6 @@ #include <type_traits> #include <utility> -#ifndef __has_feature -#define LLVM_DEFINED_HAS_FEATURE -#define __has_feature(x) 0 -#endif - namespace llvm { @@ -194,17 +189,4 @@ class is_trivially_copyable<T*> : public std::true_type { } // end namespace llvm -// If the compiler supports detecting whether a class is final, define -// an LLVM_IS_FINAL macro. If it cannot be defined properly, this -// macro will be left undefined. -#if __cplusplus >= 201402L || defined(_MSC_VER) -#define LLVM_IS_FINAL(Ty) std::is_final<Ty>() -#elif __has_feature(is_final) || LLVM_GNUC_PREREQ(4, 7, 0) -#define LLVM_IS_FINAL(Ty) __is_final(Ty) -#endif - -#ifdef LLVM_DEFINED_HAS_FEATURE -#undef __has_feature -#endif - #endif // LLVM_SUPPORT_TYPE_TRAITS_H diff --git a/include/llvm/TableGen/Automaton.td b/include/llvm/TableGen/Automaton.td new file mode 100644 index 000000000000..13ced2a0e784 --- /dev/null +++ b/include/llvm/TableGen/Automaton.td @@ -0,0 +1,95 @@ +//===- Automaton.td ----------------------------------------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the key top-level classes needed to produce a reasonably +// generic finite-state automaton. +// +//===----------------------------------------------------------------------===// + +// Define a record inheriting from GenericAutomaton to generate a reasonably +// generic finite-state automaton over a set of actions and states. +// +// This automaton is defined by: +// 1) a state space (explicit, always bits<32>). +// 2) a set of input symbols (actions, explicit) and +// 3) a transition function from state + action -> state. +// +// A theoretical automaton is defined by <Q, S, d, q0, F>: +// Q: A set of possible states. +// S: (sigma) The input alphabet. +// d: (delta) The transition function f(q in Q, s in S) -> q' in Q. +// F: The set of final (accepting) states. +// +// Because generating all possible states is tedious, we instead define the +// transition function only and crawl all reachable states starting from the +// initial state with all inputs under all transitions until termination. +// +// We define F = S, that is, all valid states are accepting. +// +// To ensure the generation of the automaton terminates, the state transitions +// are defined as a lattice (meaning every transitioned-to state is more +// specific than the transitioned-from state, for some definition of specificity). +// Concretely a transition may set one or more bits in the state that were +// previously zero to one. If any bit was not zero, the transition is invalid. +// +// Instead of defining all possible states (which would be cumbersome), the user +// provides a set of possible Transitions from state A, consuming an input +// symbol A to state B. The Transition object transforms state A to state B and +// acts as a predicate. This means the state space can be discovered by crawling +// all the possible transitions until none are valid. +// +// This automaton is considered to be nondeterministic, meaning that multiple +// transitions can occur from any (state, action) pair. The generated automaton +// is determinized, meaning that is executes in O(k) time where k is the input +// sequence length. +// +// In addition to a generated automaton that determines if a sequence of inputs +// is accepted or not, a table is emitted that allows determining a plausible +// sequence of states traversed to accept that input. +class GenericAutomaton { + // Name of a class that inherits from Transition. All records inheriting from + // this class will be considered when constructing the automaton. + string TransitionClass; + + // Names of fields within TransitionClass that define the action symbol. This + // defines the action as an N-tuple. + // + // Each symbol field can be of class, int, string or code type. + // If the type of a field is a class, the Record's name is used verbatim + // in C++ and the class name is used as the C++ type name. + // If the type of a field is a string, code or int, that is also used + // verbatim in C++. + // + // To override the C++ type name for field F, define a field called TypeOf_F. + // This should be a string that will be used verbatim in C++. + // + // As an example, to define a 2-tuple with an enum and a string, one might: + // def MyTransition : Transition { + // MyEnum S1; + // int S2; + // } + // def MyAutomaton : GenericAutomaton }{ + // let TransitionClass = "Transition"; + // let SymbolFields = ["S1", "S2"]; + // let TypeOf_S1 = "MyEnumInCxxKind"; + // } + list<string> SymbolFields; +} + +// All transitions inherit from Transition. +class Transition { + // A transition S' = T(S) is valid if, for every set bit in NewState, the + // corresponding bit in S is clear. That is: + // def T(S): + // S' = S | NewState + // return S' if S' != S else Failure + // + // The automaton generator uses this property to crawl the set of possible + // transitions from a starting state of 0b0. + bits<32> NewState; +} diff --git a/include/llvm/TableGen/Error.h b/include/llvm/TableGen/Error.h index 7c83b6298620..cf990427f577 100644 --- a/include/llvm/TableGen/Error.h +++ b/include/llvm/TableGen/Error.h @@ -18,6 +18,7 @@ namespace llvm { +void PrintNote(const Twine &Msg); void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg); void PrintWarning(ArrayRef<SMLoc> WarningLoc, const Twine &Msg); diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h index bf7f02208c28..73ed342a6101 100644 --- a/include/llvm/TableGen/Record.h +++ b/include/llvm/TableGen/Record.h @@ -1263,7 +1263,14 @@ class FieldInit : public TypedInit { FieldInit(Init *R, StringInit *FN) : TypedInit(IK_FieldInit, R->getFieldType(FN)), Rec(R), FieldName(FN) { - assert(getType() && "FieldInit with non-record type!"); +#ifndef NDEBUG + if (!getType()) { + llvm::errs() << "In Record = " << Rec->getAsString() + << ", got FieldName = " << *FieldName + << " with non-record type!\n"; + llvm_unreachable("FieldInit with non-record type!"); + } +#endif } public: @@ -1323,6 +1330,7 @@ public: void Profile(FoldingSetNodeID &ID) const; Init *getOperator() const { return Val; } + Record *getOperatorAsDef(ArrayRef<SMLoc> Loc) const; StringInit *getName() const { return ValName; } @@ -1680,10 +1688,10 @@ raw_ostream &operator<<(raw_ostream &OS, const Record &R); class RecordKeeper { friend class RecordRecTy; - using RecordMap = std::map<std::string, std::unique_ptr<Record>>; + using RecordMap = std::map<std::string, std::unique_ptr<Record>, std::less<>>; RecordMap Classes, Defs; FoldingSet<RecordRecTy> RecordTypePool; - std::map<std::string, Init *> ExtraGlobals; + std::map<std::string, Init *, std::less<>> ExtraGlobals; unsigned AnonCounter = 0; public: diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td index 45718327b4a7..4b49dfd4dd18 100644 --- a/include/llvm/Target/GenericOpcodes.td +++ b/include/llvm/Target/GenericOpcodes.td @@ -15,7 +15,9 @@ // Unary ops. //------------------------------------------------------------------------------ -class GenericInstruction : StandardPseudoInstruction; +class GenericInstruction : StandardPseudoInstruction { + let isPreISelOpcode = 1; +} // Extend the underlying scalar type of an operation, leaving the high bits // unspecified. @@ -33,6 +35,20 @@ def G_SEXT : GenericInstruction { let hasSideEffects = 0; } +// Sign extend the a value from an arbitrary bit position, copying the sign bit +// into all bits above it. This is equivalent to a shl + ashr pair with an +// appropriate shift amount. $sz is an immediate (MachineOperand::isImm() +// returns true) to allow targets to have some bitwidths legal and others +// lowered. This opcode is particularly useful if the target has sign-extension +// instructions that are cheaper than the constituent shifts as the optimizer is +// able to make decisions on whether it's better to hang on to the G_SEXT_INREG +// or to lower it and optimize the individual shifts. +def G_SEXT_INREG : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src, untyped_imm_0:$sz); + let hasSideEffects = 0; +} + // Zero extend the underlying scalar type of an operation, putting zero bits // into the newly-created space. def G_ZEXT : GenericInstruction { @@ -157,6 +173,12 @@ def G_BSWAP : GenericInstruction { let hasSideEffects = 0; } +def G_BITREVERSE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + def G_ADDRSPACE_CAST : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); @@ -175,6 +197,12 @@ def G_JUMP_TABLE : GenericInstruction { let hasSideEffects = 0; } +def G_DYN_STACKALLOC : GenericInstruction { + let OutOperandList = (outs ptype0:$dst); + let InOperandList = (ins type1:$size, i32imm:$align); + let hasSideEffects = 1; +} + //------------------------------------------------------------------------------ // Binary ops. //------------------------------------------------------------------------------ @@ -598,6 +626,15 @@ def G_FMA : GenericInstruction { let isCommutable = 0; } +/// Generic FP multiply and add. Perform a * b + c, while getting the +/// same result as the separately rounded operations, unlike G_FMA. +def G_FMAD : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2, type0:$src3); + let hasSideEffects = 0; + let isCommutable = 0; +} + // Generic FP division. def G_FDIV : GenericInstruction { let OutOperandList = (outs type0:$dst); @@ -725,7 +762,11 @@ def G_INTRINSIC_ROUND : GenericInstruction { // Memory ops //------------------------------------------------------------------------------ -// Generic load. Expects a MachineMemOperand in addition to explicit operands. +// Generic load. Expects a MachineMemOperand in addition to explicit +// operands. If the result size is larger than the memory size, the +// high bits are undefined. If the result is a vector type and larger +// than the memory size, the high elements are undefined (i.e. this is +// not a per-element, vector anyextload) def G_LOAD : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins ptype1:$addr); @@ -749,6 +790,32 @@ def G_ZEXTLOAD : GenericInstruction { let mayLoad = 1; } +// Generic indexed load. Combines a GEP with a load. $newaddr is set to $base + $offset. +// If $am is 0 (post-indexed), then the value is loaded from $base; if $am is 1 (pre-indexed) +// then the value is loaded from $newaddr. +def G_INDEXED_LOAD : GenericInstruction { + let OutOperandList = (outs type0:$dst, ptype1:$newaddr); + let InOperandList = (ins ptype1:$base, type2:$offset, unknown:$am); + let hasSideEffects = 0; + let mayLoad = 1; +} + +// Same as G_INDEXED_LOAD except that the load performed is sign-extending, as with G_SEXTLOAD. +def G_INDEXED_SEXTLOAD : GenericInstruction { + let OutOperandList = (outs type0:$dst, ptype1:$newaddr); + let InOperandList = (ins ptype1:$base, type2:$offset, unknown:$am); + let hasSideEffects = 0; + let mayLoad = 1; +} + +// Same as G_INDEXED_LOAD except that the load performed is zero-extending, as with G_ZEXTLOAD. +def G_INDEXED_ZEXTLOAD : GenericInstruction { + let OutOperandList = (outs type0:$dst, ptype1:$newaddr); + let InOperandList = (ins ptype1:$base, type2:$offset, unknown:$am); + let hasSideEffects = 0; + let mayLoad = 1; +} + // Generic store. Expects a MachineMemOperand in addition to explicit operands. def G_STORE : GenericInstruction { let OutOperandList = (outs); @@ -757,6 +824,15 @@ def G_STORE : GenericInstruction { let mayStore = 1; } +// Combines a store with a GEP. See description of G_INDEXED_LOAD for indexing behaviour. +def G_INDEXED_STORE : GenericInstruction { + let OutOperandList = (outs ptype0:$newaddr); + let InOperandList = (ins type1:$src, ptype0:$base, ptype2:$offset, + unknown:$am); + let hasSideEffects = 0; + let mayStore = 1; +} + // Generic atomic cmpxchg with internal success check. Expects a // MachineMemOperand in addition to explicit operands. def G_ATOMIC_CMPXCHG_WITH_SUCCESS : GenericInstruction { @@ -798,6 +874,8 @@ def G_ATOMICRMW_MAX : G_ATOMICRMW_OP; def G_ATOMICRMW_MIN : G_ATOMICRMW_OP; def G_ATOMICRMW_UMAX : G_ATOMICRMW_OP; def G_ATOMICRMW_UMIN : G_ATOMICRMW_OP; +def G_ATOMICRMW_FADD : G_ATOMICRMW_OP; +def G_ATOMICRMW_FSUB : G_ATOMICRMW_OP; def G_FENCE : GenericInstruction { let OutOperandList = (outs); @@ -947,9 +1025,12 @@ def G_EXTRACT_VECTOR_ELT : GenericInstruction { } // Generic shufflevector. +// +// The mask operand should be an IR Constant which exactly matches the +// corresponding mask for the IR shufflevector instruction. def G_SHUFFLE_VECTOR: GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type1:$v1, type1:$v2, type2:$mask); + let InOperandList = (ins type1:$v1, type1:$v2, unknown:$mask); let hasSideEffects = 0; } diff --git a/include/llvm/Target/GlobalISel/Combine.td b/include/llvm/Target/GlobalISel/Combine.td new file mode 100644 index 000000000000..dcac399fd693 --- /dev/null +++ b/include/llvm/Target/GlobalISel/Combine.td @@ -0,0 +1,103 @@ +//===- Combine.td - Combine rule definitions ---------------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Declare GlobalISel combine rules and provide mechanisms to opt-out. +// +//===----------------------------------------------------------------------===// + +// Common base class for GICombineRule and GICombineGroup. +class GICombine { + // See GICombineGroup. We only declare it here to make the tablegen pass + // simpler. + list<GICombine> Rules = ?; +} + +// A group of combine rules that can be added to a GICombiner or another group. +class GICombineGroup<list<GICombine> rules> : GICombine { + // The rules contained in this group. The rules in a group are flattened into + // a single list and sorted into whatever order is most efficient. However, + // they will never be re-ordered such that behaviour differs from the + // specified order. It is therefore possible to use the order of rules in this + // list to describe priorities. + let Rules = rules; +} + +// Declares a combiner helper class +class GICombinerHelper<string classname, list<GICombine> rules> + : GICombineGroup<rules> { + // The class name to use in the generated output. + string Classname = classname; + // The name of a run-time compiler option that will be generated to disable + // specific rules within this combiner. + string DisableRuleOption = ?; +} +class GICombineRule<dag defs, dag match, dag apply> : GICombine { + /// Defines the external interface of the match rule. This includes: + /// * The names of the root nodes (requires at least one) + /// See GIDefKind for details. + dag Defs = defs; + + /// Defines the things which must be true for the pattern to match + /// See GIMatchKind for details. + dag Match = match; + + /// Defines the things which happen after the decision is made to apply a + /// combine rule. + /// See GIApplyKind for details. + dag Apply = apply; +} + +/// The operator at the root of a GICombineRule.Defs dag. +def defs; + +/// All arguments of the defs operator must be subclasses of GIDefKind or +/// sub-dags whose operator is GIDefKindWithArgs. +class GIDefKind; +class GIDefKindWithArgs; +/// Declare a root node. There must be at least one of these in every combine +/// rule. +/// TODO: The plan is to elide `root` definitions and determine it from the DAG +/// itself with an overide for situations where the usual determination +/// is incorrect. +def root : GIDefKind; + +/// The operator at the root of a GICombineRule.Match dag. +def match; +/// All arguments of the match operator must be either: +/// * A subclass of GIMatchKind +/// * A subclass of GIMatchKindWithArgs +/// * A MIR code block (deprecated) +/// The GIMatchKind and GIMatchKindWithArgs cases are described in more detail +/// in their definitions below. +/// For the Instruction case, these are collected into a DAG where operand names +/// that occur multiple times introduce edges. +class GIMatchKind; +class GIMatchKindWithArgs; + +/// The operator at the root of a GICombineRule.Apply dag. +def apply; +/// All arguments of the apply operator must be subclasses of GIApplyKind, or +/// sub-dags whose operator is GIApplyKindWithArgs, or an MIR block +/// (deprecated). +class GIApplyKind; +class GIApplyKindWithArgs; + +def copy_prop : GICombineRule< + (defs root:$d), + (match [{ return Helper.matchCombineCopy(${d}); }]), + (apply [{ Helper.applyCombineCopy(${d}); }])>; +def trivial_combines : GICombineGroup<[copy_prop]>; + +// FIXME: Is there a reason this wasn't in tryCombine? I've left it out of +// all_combines because it wasn't there. +def elide_br_by_inverting_cond : GICombineRule< + (defs root:$d), + (match [{ return Helper.matchElideBrByInvertingCond(${d}); }]), + (apply [{ Helper.applyElideBrByInvertingCond(${d}); }])>; + +def all_combines : GICombineGroup<[trivial_combines]>; diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 6cc58d6521da..b846d2252b8d 100644 --- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -27,6 +27,7 @@ class GINodeEquiv<Instruction i, SDNode node> { // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel // stores this information in the MachineMemoryOperand. bit CheckMMOIsNonAtomic = 0; + bit CheckMMOIsAtomic = 0; // SelectionDAG has one node for all loads and uses predicates to // differentiate them. GlobalISel on the other hand uses separate opcodes. @@ -34,6 +35,10 @@ class GINodeEquiv<Instruction i, SDNode node> { // depending on the predicates on the node. Instruction IfSignExtend = ?; Instruction IfZeroExtend = ?; + + // SelectionDAG has one setcc for all compares. This differentiates + // for G_ICMP and G_FCMP. + Instruction IfFloatingPoint = ?; } // These are defined in the same order as the G_* instructions. @@ -46,6 +51,7 @@ def : GINodeEquiv<G_BITCAST, bitconvert>; // G_PTRTOINT - SelectionDAG has no equivalent. def : GINodeEquiv<G_CONSTANT, imm>; def : GINodeEquiv<G_FCONSTANT, fpimm>; +def : GINodeEquiv<G_IMPLICIT_DEF, undef>; def : GINodeEquiv<G_ADD, add>; def : GINodeEquiv<G_SUB, sub>; def : GINodeEquiv<G_MUL, mul>; @@ -72,6 +78,7 @@ def : GINodeEquiv<G_UITOFP, uint_to_fp>; def : GINodeEquiv<G_FADD, fadd>; def : GINodeEquiv<G_FSUB, fsub>; def : GINodeEquiv<G_FMA, fma>; +def : GINodeEquiv<G_FMAD, fmad>; def : GINodeEquiv<G_FMUL, fmul>; def : GINodeEquiv<G_FDIV, fdiv>; def : GINodeEquiv<G_FREM, frem>; @@ -85,6 +92,7 @@ def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_void>; def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>; def : GINodeEquiv<G_BR, br>; def : GINodeEquiv<G_BSWAP, bswap>; +def : GINodeEquiv<G_BITREVERSE, bitreverse>; def : GINodeEquiv<G_CTLZ, ctlz>; def : GINodeEquiv<G_CTTZ, cttz>; def : GINodeEquiv<G_CTLZ_ZERO_UNDEF, ctlz_zero_undef>; @@ -100,10 +108,15 @@ def : GINodeEquiv<G_FSQRT, fsqrt>; def : GINodeEquiv<G_FFLOOR, ffloor>; def : GINodeEquiv<G_FRINT, frint>; def : GINodeEquiv<G_FNEARBYINT, fnearbyint>; +def : GINodeEquiv<G_FCOPYSIGN, fcopysign>; def : GINodeEquiv<G_SMIN, smin>; def : GINodeEquiv<G_SMAX, smax>; def : GINodeEquiv<G_UMIN, umin>; def : GINodeEquiv<G_UMAX, umax>; +def : GINodeEquiv<G_FMINNUM, fminnum>; +def : GINodeEquiv<G_FMAXNUM, fmaxnum>; +def : GINodeEquiv<G_FMINNUM_IEEE, fminnum_ieee>; +def : GINodeEquiv<G_FMAXNUM_IEEE, fmaxnum_ieee>; // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some // complications that tablegen must take care of. For example, Predicates such @@ -117,6 +130,11 @@ def : GINodeEquiv<G_LOAD, ld> { let IfSignExtend = G_SEXTLOAD; let IfZeroExtend = G_ZEXTLOAD; } + +def : GINodeEquiv<G_ICMP, setcc> { + let IfFloatingPoint = G_FCMP; +} + // Broadly speaking G_STORE is equivalent to ISD::STORE but there are some // complications that tablegen must take care of. For example, predicates such // as isTruncStore require that this is not a perfect 1:1 mapping since a @@ -126,6 +144,11 @@ def : GINodeEquiv<G_LOAD, ld> { // G_STORE with a non-atomic MachineMemOperand. def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = 1; } +def : GINodeEquiv<G_LOAD, atomic_load> { + let CheckMMOIsNonAtomic = 0; + let CheckMMOIsAtomic = 1; +} + def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>; def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap>; def : GINodeEquiv<G_ATOMICRMW_ADD, atomic_load_add>; @@ -138,6 +161,8 @@ def : GINodeEquiv<G_ATOMICRMW_MIN, atomic_load_min>; def : GINodeEquiv<G_ATOMICRMW_MAX, atomic_load_max>; def : GINodeEquiv<G_ATOMICRMW_UMIN, atomic_load_umin>; def : GINodeEquiv<G_ATOMICRMW_UMAX, atomic_load_umax>; +def : GINodeEquiv<G_ATOMICRMW_FADD, atomic_load_fadd>; +def : GINodeEquiv<G_ATOMICRMW_FSUB, atomic_load_fsub>; def : GINodeEquiv<G_FENCE, atomic_fence>; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index d58662e128e0..dd8679661b9a 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -351,7 +351,11 @@ def interleave; // RegisterTuples instances can be used in other set operations to form // register classes and so on. This is the only way of using the generated // registers. -class RegisterTuples<list<SubRegIndex> Indices, list<dag> Regs> { +// +// RegNames may be specified to supply asm names for the generated tuples. +// If used must have the same size as the list of produced registers. +class RegisterTuples<list<SubRegIndex> Indices, list<dag> Regs, + list<string> RegNames = []> { // SubRegs - N lists of registers to be zipped up. Super-registers are // synthesized from the first element of each SubRegs list, the second // element and so on. @@ -360,6 +364,9 @@ class RegisterTuples<list<SubRegIndex> Indices, list<dag> Regs> { // SubRegIndices - N SubRegIndex instances. This provides the names of the // sub-registers in the synthesized super-registers. list<SubRegIndex> SubRegIndices = Indices; + + // List of asm names for the generated tuple registers. + list<string> RegAsmNames = RegNames; } @@ -436,6 +443,15 @@ class InstructionEncoding { bit hasCompleteDecoder = 1; } +// Allows specifying an InstructionEncoding by HwMode. If an Instruction specifies +// an EncodingByHwMode, its Inst and Size members are ignored and Ts are used +// to encode and decode based on HwMode. +class EncodingByHwMode<list<HwMode> Ms = [], list<InstructionEncoding> Ts = []> + : HwModeSelect<Ms> { + // The length of this list must be the same as the length of Ms. + list<InstructionEncoding> Objects = Ts; +} + //===----------------------------------------------------------------------===// // Instruction set description - These classes correspond to the C++ classes in // the Target/TargetInstrInfo.h file. @@ -447,6 +463,10 @@ class Instruction : InstructionEncoding { dag InOperandList; // An dag containing the MI use operand list. string AsmString = ""; // The .s format to print the instruction with. + // Allows specifying a canonical InstructionEncoding by HwMode. If non-empty, + // the Inst member of this Instruction is ignored. + EncodingByHwMode EncodingInfos; + // Pattern - Set to the DAG pattern for this instruction, if we know of one, // otherwise, uninitialized. list<dag> Pattern; @@ -472,6 +492,10 @@ class Instruction : InstructionEncoding { // Added complexity passed onto matching pattern. int AddedComplexity = 0; + // Indicates if this is a pre-isel opcode that should be + // legalized/regbankselected/selected. + bit isPreISelOpcode = 0; + // These bits capture information about the high-level semantics of the // instruction. bit isReturn = 0; // Is this instruction a return instruction? @@ -834,6 +858,7 @@ def f64imm : Operand<f64>; class TypedOperand<string Ty> : Operand<untyped> { let OperandType = Ty; bit IsPointer = 0; + bit IsImmediate = 0; } def type0 : TypedOperand<"OPERAND_GENERIC_0">; @@ -852,6 +877,12 @@ let IsPointer = 1 in { def ptype5 : TypedOperand<"OPERAND_GENERIC_5">; } +// untyped_imm is for operands where isImm() will be true. It currently has no +// special behaviour and is only used for clarity. +def untyped_imm_0 : TypedOperand<"OPERAND_GENERIC_IMM_0"> { + let IsImmediate = 1; +} + /// zero_reg definition - Special node to stand for the zero register. /// def zero_reg; diff --git a/include/llvm/Target/TargetCallingConv.td b/include/llvm/Target/TargetCallingConv.td index 1bc03cf8a49d..7b1973cc3828 100644 --- a/include/llvm/Target/TargetCallingConv.td +++ b/include/llvm/Target/TargetCallingConv.td @@ -152,6 +152,12 @@ class CCBitConvertToType<ValueType destTy> : CCAction { ValueType DestTy = destTy; } +/// CCTruncToType - If applied, this truncates the specified current value to +/// the specified type. +class CCTruncToType<ValueType destTy> : CCAction { + ValueType DestTy = destTy; +} + /// CCPassIndirect - If applied, this stores the value to stack and passes the pointer /// as normal argument. class CCPassIndirect<ValueType destTy> : CCAction { diff --git a/include/llvm/Target/TargetItinerary.td b/include/llvm/Target/TargetItinerary.td index b68ed045520c..89e5abd947d0 100644 --- a/include/llvm/Target/TargetItinerary.td +++ b/include/llvm/Target/TargetItinerary.td @@ -127,6 +127,17 @@ class ProcessorItineraries<list<FuncUnit> fu, list<Bypass> bp, list<FuncUnit> FU = fu; list<Bypass> BP = bp; list<InstrItinData> IID = iid; + // The packetizer automaton to use for this itinerary. By default all + // itineraries for a target are bundled up into the same automaton. This only + // works correctly when there are no conflicts in functional unit IDs between + // itineraries. For example, given two itineraries A<[SLOT_A]>, B<[SLOT_B]>, + // SLOT_A and SLOT_B will be assigned the same functional unit index, and + // the generated packetizer will confuse instructions referencing these slots. + // + // To avoid this, setting PacketizerNamespace to non-"" will cause this + // itinerary to be generated in a different automaton. The subtarget will need + // to declare a method "create##Namespace##DFAPacketizer()". + string PacketizerNamespace = ""; } // NoItineraries - A marker that can be used by processors without schedule diff --git a/include/llvm/Target/TargetLoweringObjectFile.h b/include/llvm/Target/TargetLoweringObjectFile.h index 3a2497bff11e..d74341b23fb1 100644 --- a/include/llvm/Target/TargetLoweringObjectFile.h +++ b/include/llvm/Target/TargetLoweringObjectFile.h @@ -191,7 +191,8 @@ public: } /// Get the target specific PC relative GOT entry relocation - virtual const MCExpr *getIndirectSymViaGOTPCRel(const MCSymbol *Sym, + virtual const MCExpr *getIndirectSymViaGOTPCRel(const GlobalValue *GV, + const MCSymbol *Sym, const MCValue &MV, int64_t Offset, MachineModuleInfo *MMI, diff --git a/include/llvm/Target/TargetMachine.h b/include/llvm/Target/TargetMachine.h index cdf9f8bfd5ea..285c0ec0fb90 100644 --- a/include/llvm/Target/TargetMachine.h +++ b/include/llvm/Target/TargetMachine.h @@ -25,7 +25,7 @@ namespace llvm { class Function; class GlobalValue; -class MachineModuleInfo; +class MachineModuleInfoWrapperPass; class Mangler; class MCAsmInfo; class MCContext; @@ -284,12 +284,13 @@ public: /// emitted. Typically this will involve several steps of code generation. /// This method should return true if emission of this file type is not /// supported, or false on success. - /// \p MMI is an optional parameter that, if set to non-nullptr, + /// \p MMIWP is an optional parameter that, if set to non-nullptr, /// will be used to set the MachineModuloInfo for this PM. - virtual bool addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &, - raw_pwrite_stream *, CodeGenFileType, - bool /*DisableVerify*/ = true, - MachineModuleInfo *MMI = nullptr) { + virtual bool + addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &, + raw_pwrite_stream *, CodeGenFileType, + bool /*DisableVerify*/ = true, + MachineModuleInfoWrapperPass *MMIWP = nullptr) { return true; } @@ -341,12 +342,13 @@ public: /// Add passes to the specified pass manager to get the specified file /// emitted. Typically this will involve several steps of code generation. - /// \p MMI is an optional parameter that, if set to non-nullptr, - /// will be used to set the MachineModuloInfofor this PM. - bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out, - raw_pwrite_stream *DwoOut, CodeGenFileType FileType, - bool DisableVerify = true, - MachineModuleInfo *MMI = nullptr) override; + /// \p MMIWP is an optional parameter that, if set to non-nullptr, + /// will be used to set the MachineModuloInfo for this PM. + bool + addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, CodeGenFileType FileType, + bool DisableVerify = true, + MachineModuleInfoWrapperPass *MMIWP = nullptr) override; /// Add passes to the specified pass manager to get machine code emitted with /// the MCJIT. This method returns true if machine code is not supported. It @@ -365,7 +367,7 @@ public: /// Adds an AsmPrinter pass to the pipeline that prints assembly or /// machine code from the MI representation. bool addAsmPrinter(PassManagerBase &PM, raw_pwrite_stream &Out, - raw_pwrite_stream *DwoOut, CodeGenFileType FileTYpe, + raw_pwrite_stream *DwoOut, CodeGenFileType FileType, MCContext &Context); /// True if the target uses physical regs at Prolog/Epilog insertion diff --git a/include/llvm/Target/TargetSchedule.td b/include/llvm/Target/TargetSchedule.td index a36d259df831..24f37e94da91 100644 --- a/include/llvm/Target/TargetSchedule.td +++ b/include/llvm/Target/TargetSchedule.td @@ -563,10 +563,10 @@ class RetireControlUnit<int bufferSize, int retirePerCycle> { // Base class for Load/StoreQueue. It is used to identify processor resources // which describe load/store queues in the LS unit. -class MemoryQueue<ProcResource PR> { - ProcResource QueueDescriptor = PR; +class MemoryQueue<ProcResourceKind PR> { + ProcResourceKind QueueDescriptor = PR; SchedMachineModel SchedModel = ?; } -class LoadQueue<ProcResource LDQueue> : MemoryQueue<LDQueue>; -class StoreQueue<ProcResource STQueue> : MemoryQueue<STQueue>; +class LoadQueue<ProcResourceKind LDQueue> : MemoryQueue<LDQueue>; +class StoreQueue<ProcResourceKind STQueue> : MemoryQueue<STQueue>; diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td index b913a054ac2c..441f3d7d118d 100644 --- a/include/llvm/Target/TargetSelectionDAG.td +++ b/include/llvm/Target/TargetSelectionDAG.td @@ -137,9 +137,12 @@ def SDTFPSignOp : SDTypeProfile<1, 2, [ // fcopysign. def SDTFPTernaryOp : SDTypeProfile<1, 3, [ // fmadd, fnmsub, etc. SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisFP<0> ]>; -def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // ctlz, cttz +def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // bitreverse SDTCisSameAs<0, 1>, SDTCisInt<0> ]>; +def SDTIntBitCountUnaryOp : SDTypeProfile<1, 1, [ // ctlz, cttz + SDTCisInt<0>, SDTCisInt<1> +]>; def SDTIntExtendOp : SDTypeProfile<1, 1, [ // sext, zext, anyext SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<1, 0>, SDTCisSameNumEltsAs<0, 1> ]>; @@ -239,6 +242,9 @@ def SDTVecExtract : SDTypeProfile<1, 2, [ // vector extract def SDTVecInsert : SDTypeProfile<1, 3, [ // vector insert SDTCisEltOfVec<2, 1>, SDTCisSameAs<0, 1>, SDTCisPtrTy<3> ]>; +def SDTVecReduce : SDTypeProfile<1, 1, [ // vector reduction + SDTCisInt<0>, SDTCisVec<1> +]>; def SDTSubVecExtract : SDTypeProfile<1, 2, [// subvector extract SDTCisSubVecOfVec<0,1>, SDTCisInt<2> @@ -393,6 +399,7 @@ def usubsat : SDNode<"ISD::USUBSAT" , SDTIntBinOp>; def smulfix : SDNode<"ISD::SMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>; def smulfixsat : SDNode<"ISD::SMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>; def umulfix : SDNode<"ISD::UMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>; +def umulfixsat : SDNode<"ISD::UMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>; def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>; def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>; @@ -401,11 +408,11 @@ def zext_invec : SDNode<"ISD::ZERO_EXTEND_VECTOR_INREG", SDTExtInvec>; def abs : SDNode<"ISD::ABS" , SDTIntUnaryOp>; def bitreverse : SDNode<"ISD::BITREVERSE" , SDTIntUnaryOp>; def bswap : SDNode<"ISD::BSWAP" , SDTIntUnaryOp>; -def ctlz : SDNode<"ISD::CTLZ" , SDTIntUnaryOp>; -def cttz : SDNode<"ISD::CTTZ" , SDTIntUnaryOp>; -def ctpop : SDNode<"ISD::CTPOP" , SDTIntUnaryOp>; -def ctlz_zero_undef : SDNode<"ISD::CTLZ_ZERO_UNDEF", SDTIntUnaryOp>; -def cttz_zero_undef : SDNode<"ISD::CTTZ_ZERO_UNDEF", SDTIntUnaryOp>; +def ctlz : SDNode<"ISD::CTLZ" , SDTIntBitCountUnaryOp>; +def cttz : SDNode<"ISD::CTTZ" , SDTIntBitCountUnaryOp>; +def ctpop : SDNode<"ISD::CTPOP" , SDTIntBitCountUnaryOp>; +def ctlz_zero_undef : SDNode<"ISD::CTLZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>; +def cttz_zero_undef : SDNode<"ISD::CTTZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>; def sext : SDNode<"ISD::SIGN_EXTEND", SDTIntExtendOp>; def zext : SDNode<"ISD::ZERO_EXTEND", SDTIntExtendOp>; def anyext : SDNode<"ISD::ANY_EXTEND" , SDTIntExtendOp>; @@ -415,6 +422,12 @@ def addrspacecast : SDNode<"ISD::ADDRSPACECAST", SDTUnaryOp>; def extractelt : SDNode<"ISD::EXTRACT_VECTOR_ELT", SDTVecExtract>; def insertelt : SDNode<"ISD::INSERT_VECTOR_ELT", SDTVecInsert>; +def vecreduce_add : SDNode<"ISD::VECREDUCE_ADD", SDTVecReduce>; +def vecreduce_smax : SDNode<"ISD::VECREDUCE_SMAX", SDTVecReduce>; +def vecreduce_umax : SDNode<"ISD::VECREDUCE_UMAX", SDTVecReduce>; +def vecreduce_smin : SDNode<"ISD::VECREDUCE_SMIN", SDTVecReduce>; +def vecreduce_umin : SDNode<"ISD::VECREDUCE_UMIN", SDTVecReduce>; + def fadd : SDNode<"ISD::FADD" , SDTFPBinOp, [SDNPCommutative]>; def fsub : SDNode<"ISD::FSUB" , SDTFPBinOp>; def fmul : SDNode<"ISD::FMUL" , SDTFPBinOp, [SDNPCommutative]>; @@ -493,12 +506,20 @@ def strict_flog2 : SDNode<"ISD::STRICT_FLOG2", SDTFPUnaryOp, [SDNPHasChain]>; def strict_frint : SDNode<"ISD::STRICT_FRINT", SDTFPUnaryOp, [SDNPHasChain]>; +def strict_lrint : SDNode<"ISD::STRICT_LRINT", + SDTFPToIntOp, [SDNPHasChain]>; +def strict_llrint : SDNode<"ISD::STRICT_LLRINT", + SDTFPToIntOp, [SDNPHasChain]>; def strict_fnearbyint : SDNode<"ISD::STRICT_FNEARBYINT", SDTFPUnaryOp, [SDNPHasChain]>; def strict_fceil : SDNode<"ISD::STRICT_FCEIL", SDTFPUnaryOp, [SDNPHasChain]>; def strict_ffloor : SDNode<"ISD::STRICT_FFLOOR", SDTFPUnaryOp, [SDNPHasChain]>; +def strict_lround : SDNode<"ISD::STRICT_LROUND", + SDTFPToIntOp, [SDNPHasChain]>; +def strict_llround : SDNode<"ISD::STRICT_LLROUND", + SDTFPToIntOp, [SDNPHasChain]>; def strict_fround : SDNode<"ISD::STRICT_FROUND", SDTFPUnaryOp, [SDNPHasChain]>; def strict_ftrunc : SDNode<"ISD::STRICT_FTRUNC", @@ -513,6 +534,10 @@ def strict_fpround : SDNode<"ISD::STRICT_FP_ROUND", SDTFPRoundOp, [SDNPHasChain]>; def strict_fpextend : SDNode<"ISD::STRICT_FP_EXTEND", SDTFPExtendOp, [SDNPHasChain]>; +def strict_fp_to_sint : SDNode<"ISD::STRICT_FP_TO_SINT", + SDTFPToIntOp, [SDNPHasChain]>; +def strict_fp_to_uint : SDNode<"ISD::STRICT_FP_TO_UINT", + SDTFPToIntOp, [SDNPHasChain]>; def setcc : SDNode<"ISD::SETCC" , SDTSetCC>; def select : SDNode<"ISD::SELECT" , SDTSelect>; @@ -638,16 +663,32 @@ def assertzext : SDNode<"ISD::AssertZext", SDT_assertext>; //===----------------------------------------------------------------------===// // Selection DAG Condition Codes -class CondCode; // ISD::CondCode enums -def SETOEQ : CondCode; def SETOGT : CondCode; -def SETOGE : CondCode; def SETOLT : CondCode; def SETOLE : CondCode; -def SETONE : CondCode; def SETO : CondCode; def SETUO : CondCode; -def SETUEQ : CondCode; def SETUGT : CondCode; def SETUGE : CondCode; -def SETULT : CondCode; def SETULE : CondCode; def SETUNE : CondCode; - -def SETEQ : CondCode; def SETGT : CondCode; def SETGE : CondCode; -def SETLT : CondCode; def SETLE : CondCode; def SETNE : CondCode; - +class CondCode<string fcmpName = "", string icmpName = ""> { + string ICmpPredicate = icmpName; + string FCmpPredicate = fcmpName; +} + +// ISD::CondCode enums, and mapping to CmpInst::Predicate names +def SETOEQ : CondCode<"FCMP_OEQ">; +def SETOGT : CondCode<"FCMP_OGT">; +def SETOGE : CondCode<"FCMP_OGE">; +def SETOLT : CondCode<"FCMP_OLT">; +def SETOLE : CondCode<"FCMP_OLE">; +def SETONE : CondCode<"FCMP_ONE">; +def SETO : CondCode<"FCMP_ORD">; +def SETUO : CondCode<"FCMP_UNO">; +def SETUEQ : CondCode<"FCMP_UEQ">; +def SETUGT : CondCode<"FCMP_UGT", "ICMP_UGT">; +def SETUGE : CondCode<"FCMP_UGE", "ICMP_UGE">; +def SETULT : CondCode<"FCMP_ULT", "ICMP_ULT">; +def SETULE : CondCode<"FCMP_ULE", "ICMP_ULE">; +def SETUNE : CondCode<"FCMP_UNE">; +def SETEQ : CondCode<"", "ICMP_EQ">; +def SETGT : CondCode<"", "ICMP_SGT">; +def SETGE : CondCode<"", "ICMP_SGE">; +def SETLT : CondCode<"", "ICMP_SLT">; +def SETLE : CondCode<"", "ICMP_SLE">; +def SETNE : CondCode<"", "ICMP_NE">; //===----------------------------------------------------------------------===// // Selection DAG Node Transformation Functions. @@ -741,6 +782,10 @@ class PatFrags<dag ops, list<dag> frags, code pred = [{}], // If this empty, accept any address space. list<int> AddressSpaces = ?; + // cast<MemSDNode>(N)->getAlignment() >= + // If this is empty, accept any alignment. + int MinAlignment = ?; + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Monotonic bit IsAtomicOrderingMonotonic = ?; // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Acquire @@ -766,8 +811,6 @@ class PatFrags<dag ops, list<dag> frags, code pred = [{}], // cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>; // cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>; ValueType ScalarMemoryVT = ?; - - // TODO: Add alignment } // PatFrag - A version of PatFrags matching only a single fragment. @@ -813,6 +856,11 @@ class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm, bit IsAPFloat = 0; } +// Convenience wrapper for ImmLeaf to use timm/TargetConstant instead +// of imm/Constant. +class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm, + SDNode ImmNode = timm> : ImmLeaf<vt, pred, xform, ImmNode>; + // An ImmLeaf except that Imm is an APInt. This is useful when you need to // zero-extend the immediate instead of sign-extend it. // @@ -1111,6 +1159,16 @@ def pre_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset), let IsStore = 1; let MemoryVT = f32; } +def pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset), + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let ScalarMemoryVT = i8; +} +def pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset), + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let ScalarMemoryVT = i16; +} def post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset), (istore node:$val, node:$ptr, node:$offset), [{ @@ -1148,14 +1206,26 @@ def post_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset), let IsStore = 1; let MemoryVT = f32; } +def post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset), + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let ScalarMemoryVT = i8; +} +def post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset), + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let ScalarMemoryVT = i16; +} -def nonvolatile_load : PatFrag<(ops node:$ptr), - (load node:$ptr), [{ - return !cast<LoadSDNode>(N)->isVolatile(); +// TODO: Split these into volatile and unordered flavors to enable +// selectively legal optimizations for each. (See D66309) +def simple_load : PatFrag<(ops node:$ptr), + (load node:$ptr), [{ + return cast<LoadSDNode>(N)->isSimple(); }]>; -def nonvolatile_store : PatFrag<(ops node:$val, node:$ptr), - (store node:$val, node:$ptr), [{ - return !cast<StoreSDNode>(N)->isVolatile(); +def simple_store : PatFrag<(ops node:$val, node:$ptr), + (store node:$val, node:$ptr), [{ + return cast<StoreSDNode>(N)->isSimple(); }]>; // nontemporal store fragments. @@ -1277,6 +1347,12 @@ def any_flog2 : PatFrags<(ops node:$src), def any_frint : PatFrags<(ops node:$src), [(strict_frint node:$src), (frint node:$src)]>; +def any_lrint : PatFrags<(ops node:$src), + [(strict_lrint node:$src), + (lrint node:$src)]>; +def any_llrint : PatFrags<(ops node:$src), + [(strict_llrint node:$src), + (llrint node:$src)]>; def any_fnearbyint : PatFrags<(ops node:$src), [(strict_fnearbyint node:$src), (fnearbyint node:$src)]>; @@ -1286,6 +1362,12 @@ def any_fceil : PatFrags<(ops node:$src), def any_ffloor : PatFrags<(ops node:$src), [(strict_ffloor node:$src), (ffloor node:$src)]>; +def any_lround : PatFrags<(ops node:$src), + [(strict_lround node:$src), + (lround node:$src)]>; +def any_llround : PatFrags<(ops node:$src), + [(strict_llround node:$src), + (llround node:$src)]>; def any_fround : PatFrags<(ops node:$src), [(strict_fround node:$src), (fround node:$src)]>; @@ -1310,6 +1392,12 @@ def any_extloadf32 : PatFrags<(ops node:$ptr), def any_extloadf64 : PatFrags<(ops node:$ptr), [(strict_extloadf64 node:$ptr), (extloadf64 node:$ptr)]>; +def any_fp_to_sint : PatFrags<(ops node:$src), + [(strict_fp_to_sint node:$src), + (fp_to_sint node:$src)]>; +def any_fp_to_uint : PatFrags<(ops node:$src), + [(strict_fp_to_uint node:$src), + (fp_to_uint node:$src)]>; multiclass binary_atomic_op_ord<SDNode atomic_op> { def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), @@ -1367,26 +1455,26 @@ multiclass ternary_atomic_op_ord<SDNode atomic_op> { } } -multiclass binary_atomic_op<SDNode atomic_op> { +multiclass binary_atomic_op<SDNode atomic_op, bit IsInt = 1> { def _8 : PatFrag<(ops node:$ptr, node:$val), (atomic_op node:$ptr, node:$val)> { let IsAtomic = 1; - let MemoryVT = i8; + let MemoryVT = !if(IsInt, i8, ?); } def _16 : PatFrag<(ops node:$ptr, node:$val), (atomic_op node:$ptr, node:$val)> { let IsAtomic = 1; - let MemoryVT = i16; + let MemoryVT = !if(IsInt, i16, f16); } def _32 : PatFrag<(ops node:$ptr, node:$val), (atomic_op node:$ptr, node:$val)> { let IsAtomic = 1; - let MemoryVT = i32; + let MemoryVT = !if(IsInt, i32, f32); } def _64 : PatFrag<(ops node:$ptr, node:$val), (atomic_op node:$ptr, node:$val)> { let IsAtomic = 1; - let MemoryVT = i64; + let MemoryVT = !if(IsInt, i64, f64); } defm NAME#_8 : binary_atomic_op_ord<atomic_op>; diff --git a/include/llvm/TextAPI/MachO/Architecture.h b/include/llvm/TextAPI/MachO/Architecture.h index 055baeb0c0f0..3898cbada68f 100644 --- a/include/llvm/TextAPI/MachO/Architecture.h +++ b/include/llvm/TextAPI/MachO/Architecture.h @@ -14,6 +14,7 @@ #define LLVM_TEXTAPI_MACHO_ARCHITECTURE_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" #include "llvm/Support/raw_ostream.h" namespace llvm { @@ -39,6 +40,9 @@ StringRef getArchitectureName(Architecture Arch); /// Convert an architecture slice to a CPU Type and Subtype pair. std::pair<uint32_t, uint32_t> getCPUTypeFromArchitecture(Architecture Arch); +/// Convert a target to an architecture slice. +Architecture mapToArchitecture(const llvm::Triple &Target); + raw_ostream &operator<<(raw_ostream &OS, Architecture Arch); } // end namespace MachO. diff --git a/include/llvm/TextAPI/MachO/ArchitectureSet.h b/include/llvm/TextAPI/MachO/ArchitectureSet.h index d8dfc7f1af21..6e4ede6275b4 100644 --- a/include/llvm/TextAPI/MachO/ArchitectureSet.h +++ b/include/llvm/TextAPI/MachO/ArchitectureSet.h @@ -59,6 +59,10 @@ public: ArchSetType rawValue() const { return ArchSet; } + bool hasX86() const { + return has(AK_i386) || has(AK_x86_64) || has(AK_x86_64h); + } + template <typename Ty> class arch_iterator : public std::iterator<std::forward_iterator_tag, Architecture, size_t> { diff --git a/include/llvm/TextAPI/MachO/InterfaceFile.h b/include/llvm/TextAPI/MachO/InterfaceFile.h index e722449d52f1..bd434e04b693 100644 --- a/include/llvm/TextAPI/MachO/InterfaceFile.h +++ b/include/llvm/TextAPI/MachO/InterfaceFile.h @@ -26,21 +26,13 @@ #include "llvm/TextAPI/MachO/Architecture.h" #include "llvm/TextAPI/MachO/ArchitectureSet.h" #include "llvm/TextAPI/MachO/PackedVersion.h" +#include "llvm/TextAPI/MachO/Platform.h" #include "llvm/TextAPI/MachO/Symbol.h" +#include "llvm/TextAPI/MachO/Target.h" namespace llvm { namespace MachO { -/// Defines the list of MachO platforms. -enum class PlatformKind : unsigned { - unknown, - macOS = MachO::PLATFORM_MACOS, - iOS = MachO::PLATFORM_IOS, - tvOS = MachO::PLATFORM_TVOS, - watchOS = MachO::PLATFORM_WATCHOS, - bridgeOS = MachO::PLATFORM_BRIDGEOS, -}; - /// Defines a list of Objective-C constraints. enum class ObjCConstraintType : unsigned { /// No constraint. @@ -75,6 +67,9 @@ enum FileType : unsigned { /// Text-based stub file (.tbd) version 3.0 TBD_V3 = 1U << 2, + /// Text-based stub file (.tbd) version 4.0 + TBD_V4 = 1U << 3, + All = ~0U, LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All), @@ -89,29 +84,42 @@ public: InterfaceFileRef(StringRef InstallName) : InstallName(InstallName) {} - InterfaceFileRef(StringRef InstallName, ArchitectureSet Archs) - : InstallName(InstallName), Architectures(Archs) {} + InterfaceFileRef(StringRef InstallName, const TargetList Targets) + : InstallName(InstallName), Targets(std::move(Targets)) {} StringRef getInstallName() const { return InstallName; }; - void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; } - ArchitectureSet getArchitectures() const { return Architectures; } - bool hasArchitecture(Architecture Arch) const { - return Architectures.has(Arch); + + void addTarget(const Target &Target); + template <typename RangeT> void addTargets(RangeT &&Targets) { + for (const auto &Target : Targets) + addTarget(Target(Target)); } + using const_target_iterator = TargetList::const_iterator; + using const_target_range = llvm::iterator_range<const_target_iterator>; + const_target_range targets() const { return {Targets}; } + + ArchitectureSet getArchitectures() const { + return mapToArchitectureSet(Targets); + } + + PlatformSet getPlatforms() const { return mapToPlatformSet(Targets); } + bool operator==(const InterfaceFileRef &O) const { - return std::tie(InstallName, Architectures) == - std::tie(O.InstallName, O.Architectures); + return std::tie(InstallName, Targets) == std::tie(O.InstallName, O.Targets); + } + + bool operator!=(const InterfaceFileRef &O) const { + return std::tie(InstallName, Targets) != std::tie(O.InstallName, O.Targets); } bool operator<(const InterfaceFileRef &O) const { - return std::tie(InstallName, Architectures) < - std::tie(O.InstallName, O.Architectures); + return std::tie(InstallName, Targets) < std::tie(O.InstallName, O.Targets); } private: std::string InstallName; - ArchitectureSet Architectures; + TargetList Targets; }; } // end namespace MachO. @@ -170,27 +178,43 @@ public: /// \return The file type. FileType getFileType() const { return FileKind; } - /// Set the platform. - void setPlatform(PlatformKind Platform_) { Platform = Platform_; } + /// Get the architectures. + /// + /// \return The applicable architectures. + ArchitectureSet getArchitectures() const { + return mapToArchitectureSet(Targets); + } - /// Get the platform. - PlatformKind getPlatform() const { return Platform; } + /// Get the platforms. + /// + /// \return The applicable platforms. + PlatformSet getPlatforms() const { return mapToPlatformSet(Targets); } - /// Specify the set of supported architectures by this file. - void setArchitectures(ArchitectureSet Architectures_) { - Architectures = Architectures_; - } + /// Set and add target. + /// + /// \param Target the target to add into. + void addTarget(const Target &Target); - /// Add the set of supported architectures by this file. - void addArchitectures(ArchitectureSet Architectures_) { - Architectures |= Architectures_; + /// Set and add targets. + /// + /// Add the subset of llvm::triples that is supported by Tapi + /// + /// \param Targets the collection of targets. + template <typename RangeT> void addTargets(RangeT &&Targets) { + for (const auto &Target_ : Targets) + addTarget(Target(Target_)); } - /// Add supported architecture by this file.. - void addArch(Architecture Arch) { Architectures.set(Arch); } + using const_target_iterator = TargetList::const_iterator; + using const_target_range = llvm::iterator_range<const_target_iterator>; + const_target_range targets() const { return {Targets}; } - /// Get the set of supported architectures. - ArchitectureSet getArchitectures() const { return Architectures; } + using const_filtered_target_iterator = + llvm::filter_iterator<const_target_iterator, + std::function<bool(const Target &)>>; + using const_filtered_target_range = + llvm::iterator_range<const_filtered_target_iterator>; + const_filtered_target_range targets(ArchitectureSet Archs) const; /// Set the install name of the library. void setInstallName(StringRef InstallName_) { InstallName = InstallName_; } @@ -244,11 +268,18 @@ public: /// Check if this file was generated during InstallAPI. bool isInstallAPI() const { return IsInstallAPI; } - /// Set the parent umbrella framework. - void setParentUmbrella(StringRef Parent) { ParentUmbrella = Parent; } + /// Set the parent umbrella frameworks. + /// \param Target_ The target applicable to Parent + /// \param Parent The name of Parent + void addParentUmbrella(const Target &Target_, StringRef Parent); + const std::vector<std::pair<Target, std::string>> &umbrellas() const { + return ParentUmbrellas; + } /// Get the parent umbrella framework. - StringRef getParentUmbrella() const { return ParentUmbrella; } + const std::vector<std::pair<Target, std::string>> getParentUmbrellas() const { + return ParentUmbrellas; + } /// Add an allowable client. /// @@ -257,9 +288,9 @@ public: /// that is being generated needs to match one of the allowable clients or the /// linker refuses to link this library. /// - /// \param Name The name of the client that is allowed to link this library. - /// \param Architectures The set of architecture for which this applies. - void addAllowableClient(StringRef Name, ArchitectureSet Architectures); + /// \param InstallName The name of the client that is allowed to link this library. + /// \param Target The target triple for which this applies. + void addAllowableClient(StringRef InstallName, const Target &Target); /// Get the list of allowable clients. /// @@ -271,9 +302,8 @@ public: /// Add a re-exported library. /// /// \param InstallName The name of the library to re-export. - /// \param Architectures The set of architecture for which this applies. - void addReexportedLibrary(StringRef InstallName, - ArchitectureSet Architectures); + /// \param Target The target triple for which this applies. + void addReexportedLibrary(StringRef InstallName, const Target &Target); /// Get the list of re-exported libraries. /// @@ -282,27 +312,27 @@ public: return ReexportedLibraries; } - /// Add an architecture/UUID pair. + /// Add an Target/UUID pair. /// - /// \param Arch The architecture for which this applies. + /// \param Target The target triple for which this applies. /// \param UUID The UUID of the library for the specified architecture. - void addUUID(Architecture Arch, StringRef UUID); + void addUUID(const Target &Target, StringRef UUID); - /// Add an architecture/UUID pair. + /// Add an Target/UUID pair. /// - /// \param Arch The architecture for which this applies. + /// \param Target The target triple for which this applies. /// \param UUID The UUID of the library for the specified architecture. - void addUUID(Architecture Arch, uint8_t UUID[16]); + void addUUID(const Target &Target, uint8_t UUID[16]); - /// Get the list of architecture/UUID pairs. + /// Get the list of Target/UUID pairs. /// - /// \return Returns a list of architecture/UUID pairs. - const std::vector<std::pair<Architecture, std::string>> &uuids() const { + /// \return Returns a list of Target/UUID pairs. + const std::vector<std::pair<Target, std::string>> &uuids() const { return UUIDs; } /// Add a symbol to the symbols list or extend an existing one. - void addSymbol(SymbolKind Kind, StringRef Name, ArchitectureSet Architectures, + void addSymbol(SymbolKind Kind, StringRef Name, const TargetList &Targets, SymbolFlags Flags = SymbolFlags::None); using SymbolMapType = DenseMap<SymbolsMapKey, Symbol *>; @@ -320,84 +350,35 @@ public: reference operator*() const { return I->second; } pointer operator->() const { return I->second; } }; - using const_symbol_range = iterator_range<const_symbol_iterator>; - - // Custom iterator to return only exported symbols. - struct const_export_iterator - : public iterator_adaptor_base< - const_export_iterator, const_symbol_iterator, - std::forward_iterator_tag, const Symbol *> { - const_symbol_iterator _end; - - void skipToNextSymbol() { - while (I != _end && I->isUndefined()) - ++I; - } - - const_export_iterator() = default; - template <typename U> - const_export_iterator(U &&it, U &&end) - : iterator_adaptor_base(std::forward<U &&>(it)), - _end(std::forward<U &&>(end)) { - skipToNextSymbol(); - } - - const_export_iterator &operator++() { - ++I; - skipToNextSymbol(); - return *this; - } - - const_export_iterator operator++(int) { - const_export_iterator tmp(*this); - ++(*this); - return tmp; - } - }; - using const_export_range = llvm::iterator_range<const_export_iterator>; - - // Custom iterator to return only undefined symbols. - struct const_undefined_iterator - : public iterator_adaptor_base< - const_undefined_iterator, const_symbol_iterator, - std::forward_iterator_tag, const Symbol *> { - const_symbol_iterator _end; - void skipToNextSymbol() { - while (I != _end && !I->isUndefined()) - ++I; - } + using const_symbol_range = iterator_range<const_symbol_iterator>; - const_undefined_iterator() = default; - template <typename U> - const_undefined_iterator(U &&it, U &&end) - : iterator_adaptor_base(std::forward<U &&>(it)), - _end(std::forward<U &&>(end)) { - skipToNextSymbol(); - } - - const_undefined_iterator &operator++() { - ++I; - skipToNextSymbol(); - return *this; - } - - const_undefined_iterator operator++(int) { - const_undefined_iterator tmp(*this); - ++(*this); - return tmp; - } - }; - using const_undefined_range = llvm::iterator_range<const_undefined_iterator>; + using const_filtered_symbol_iterator = + filter_iterator<const_symbol_iterator, + std::function<bool(const Symbol *)>>; + using const_filtered_symbol_range = + iterator_range<const_filtered_symbol_iterator>; const_symbol_range symbols() const { return {Symbols.begin(), Symbols.end()}; } - const_export_range exports() const { - return {{Symbols.begin(), Symbols.end()}, {Symbols.end(), Symbols.end()}}; + + const_filtered_symbol_range exports() const { + std::function<bool(const Symbol *)> fn = [](const Symbol *Symbol) { + return !Symbol->isUndefined(); + }; + return make_filter_range( + make_range<const_symbol_iterator>({Symbols.begin()}, {Symbols.end()}), + fn); } - const_undefined_range undefineds() const { - return {{Symbols.begin(), Symbols.end()}, {Symbols.end(), Symbols.end()}}; + + const_filtered_symbol_range undefineds() const { + std::function<bool(const Symbol *)> fn = [](const Symbol *Symbol) { + return Symbol->isUndefined(); + }; + return make_filter_range( + make_range<const_symbol_iterator>({Symbols.begin()}, {Symbols.end()}), + fn); } private: @@ -411,10 +392,9 @@ private: return StringRef(reinterpret_cast<const char *>(Ptr), String.size()); } + TargetList Targets; std::string Path; FileType FileKind; - PlatformKind Platform; - ArchitectureSet Architectures; std::string InstallName; PackedVersion CurrentVersion; PackedVersion CompatibilityVersion; @@ -423,10 +403,10 @@ private: bool IsAppExtensionSafe{false}; bool IsInstallAPI{false}; ObjCConstraintType ObjcConstraint = ObjCConstraintType::None; - std::string ParentUmbrella; + std::vector<std::pair<Target, std::string>> ParentUmbrellas; std::vector<InterfaceFileRef> AllowableClients; std::vector<InterfaceFileRef> ReexportedLibraries; - std::vector<std::pair<Architecture, std::string>> UUIDs; + std::vector<std::pair<Target, std::string>> UUIDs; SymbolMapType Symbols; }; diff --git a/include/llvm/TextAPI/MachO/Platform.h b/include/llvm/TextAPI/MachO/Platform.h new file mode 100644 index 000000000000..a22aae9b7dce --- /dev/null +++ b/include/llvm/TextAPI/MachO/Platform.h @@ -0,0 +1,45 @@ +//===- llvm/TextAPI/MachO/Platform.h - Platform -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines the Platforms supported by Tapi and helpers. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TEXTAPI_MACHO_PLATFORM_H +#define LLVM_TEXTAPI_MACHO_PLATFORM_H + +#include "llvm/ADT/SmallSet.h" +#include "llvm/BinaryFormat/MachO.h" + +namespace llvm { +namespace MachO { + +/// Defines the list of MachO platforms. +enum class PlatformKind : unsigned { + unknown, + macOS = MachO::PLATFORM_MACOS, + iOS = MachO::PLATFORM_IOS, + tvOS = MachO::PLATFORM_TVOS, + watchOS = MachO::PLATFORM_WATCHOS, + bridgeOS = MachO::PLATFORM_BRIDGEOS, + macCatalyst = MachO::PLATFORM_MACCATALYST, + iOSSimulator = MachO::PLATFORM_IOSSIMULATOR, + tvOSSimulator = MachO::PLATFORM_TVOSSIMULATOR, + watchOSSimulator = MachO::PLATFORM_WATCHOSSIMULATOR +}; + +using PlatformSet = SmallSet<PlatformKind, 3>; + +PlatformKind mapToPlatformKind(PlatformKind Platform, bool WantSim); +PlatformKind mapToPlatformKind(const Triple &Target); +PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets); +StringRef getPlatformName(PlatformKind Platform); + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_PLATFORM_H
\ No newline at end of file diff --git a/include/llvm/TextAPI/MachO/Symbol.h b/include/llvm/TextAPI/MachO/Symbol.h index 3c7ff5e0f4ea..1b1632c599c4 100644 --- a/include/llvm/TextAPI/MachO/Symbol.h +++ b/include/llvm/TextAPI/MachO/Symbol.h @@ -14,6 +14,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TextAPI/MachO/ArchitectureSet.h" +#include "llvm/TextAPI/MachO/Target.h" namespace llvm { namespace MachO { @@ -37,7 +38,10 @@ enum class SymbolFlags : uint8_t { /// Undefined Undefined = 1U << 3, - LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Undefined), + /// Rexported + Rexported = 1U << 4, + + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Rexported), }; // clang-format on @@ -49,16 +53,18 @@ enum class SymbolKind : uint8_t { ObjectiveCInstanceVariable, }; +using TargetList = SmallVector<Target, 5>; class Symbol { public: - constexpr Symbol(SymbolKind Kind, StringRef Name, - ArchitectureSet Architectures, SymbolFlags Flags) - : Name(Name), Architectures(Architectures), Kind(Kind), Flags(Flags) {} + Symbol(SymbolKind Kind, StringRef Name, TargetList Targets, SymbolFlags Flags) + : Name(Name), Targets(std::move(Targets)), Kind(Kind), Flags(Flags) {} + void addTarget(Target target) { Targets.emplace_back(target); } SymbolKind getKind() const { return Kind; } StringRef getName() const { return Name; } - ArchitectureSet getArchitectures() const { return Architectures; } - void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; } + ArchitectureSet getArchitectures() const { + return mapToArchitectureSet(Targets); + } SymbolFlags getFlags() const { return Flags; } bool isWeakDefined() const { @@ -78,6 +84,21 @@ public: return (Flags & SymbolFlags::Undefined) == SymbolFlags::Undefined; } + bool isReexported() const { + return (Flags & SymbolFlags::Rexported) == SymbolFlags::Rexported; + } + + using const_target_iterator = TargetList::const_iterator; + using const_target_range = llvm::iterator_range<const_target_iterator>; + const_target_range targets() const { return {Targets}; } + + using const_filtered_target_iterator = + llvm::filter_iterator<const_target_iterator, + std::function<bool(const Target &)>>; + using const_filtered_target_range = + llvm::iterator_range<const_filtered_target_iterator>; + const_filtered_target_range targets(ArchitectureSet architectures) const; + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void dump(raw_ostream &OS) const; void dump() const { dump(llvm::errs()); } @@ -85,7 +106,7 @@ public: private: StringRef Name; - ArchitectureSet Architectures; + TargetList Targets; SymbolKind Kind; SymbolFlags Flags; }; diff --git a/include/llvm/TextAPI/MachO/Target.h b/include/llvm/TextAPI/MachO/Target.h new file mode 100644 index 000000000000..5fe44cb7d366 --- /dev/null +++ b/include/llvm/TextAPI/MachO/Target.h @@ -0,0 +1,68 @@ +//===- llvm/TextAPI/Target.h - TAPI Target ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_TARGET_H +#define LLVM_TEXTAPI_MACHO_TARGET_H + +#include "llvm/ADT/Triple.h" +#include "llvm/Support/Error.h" +#include "llvm/TextAPI/MachO/Architecture.h" +#include "llvm/TextAPI/MachO/ArchitectureSet.h" +#include "llvm/TextAPI/MachO/Platform.h" + +namespace llvm { +namespace MachO { + +// This is similar to a llvm Triple, but the triple doesn't have all the +// information we need. For example there is no enum value for x86_64h. The +// only way to get that information is to parse the triple string. +class Target { +public: + Target() = default; + Target(Architecture Arch, PlatformKind Platform) + : Arch(Arch), Platform(Platform) {} + explicit Target(const llvm::Triple &Triple) + : Arch(mapToArchitecture(Triple)), Platform(mapToPlatformKind(Triple)) {} + + static llvm::Expected<Target> create(StringRef Target); + + operator std::string() const; + + Architecture Arch; + PlatformKind Platform; +}; + +inline bool operator==(const Target &LHS, const Target &RHS) { + return std::tie(LHS.Arch, LHS.Platform) == std::tie(RHS.Arch, RHS.Platform); +} + +inline bool operator!=(const Target &LHS, const Target &RHS) { + return std::tie(LHS.Arch, LHS.Platform) != std::tie(RHS.Arch, RHS.Platform); +} + +inline bool operator<(const Target &LHS, const Target &RHS) { + return std::tie(LHS.Arch, LHS.Platform) < std::tie(RHS.Arch, RHS.Platform); +} + +inline bool operator==(const Target &LHS, const Architecture &RHS) { + return LHS.Arch == RHS; +} + +inline bool operator!=(const Target &LHS, const Architecture &RHS) { + return LHS.Arch != RHS; +} + +PlatformSet mapToPlatformSet(ArrayRef<Target> Targets); +ArchitectureSet mapToArchitectureSet(ArrayRef<Target> Targets); + +raw_ostream &operator<<(raw_ostream &OS, const Target &Target); + +} // namespace MachO +} // namespace llvm + +#endif // LLVM_TEXTAPI_MACHO_TARGET_H diff --git a/include/llvm/TextAPI/MachO/TextAPIReader.h b/include/llvm/TextAPI/MachO/TextAPIReader.h index 6d9c09de5294..c551f0454e8e 100644 --- a/include/llvm/TextAPI/MachO/TextAPIReader.h +++ b/include/llvm/TextAPI/MachO/TextAPIReader.h @@ -20,10 +20,7 @@ class InterfaceFile; class TextAPIReader { public: static Expected<std::unique_ptr<InterfaceFile>> - get(std::unique_ptr<MemoryBuffer> InputBuffer); - - static Expected<std::unique_ptr<InterfaceFile>> - getUnmanaged(llvm::MemoryBuffer *InputBuffer); + get(MemoryBufferRef InputBuffer); TextAPIReader() = delete; }; diff --git a/include/llvm/Transforms/IPO/Attributor.h b/include/llvm/Transforms/IPO/Attributor.h index 5dbe21ac5e4e..3dbe0fcd76ea 100644 --- a/include/llvm/Transforms/IPO/Attributor.h +++ b/include/llvm/Transforms/IPO/Attributor.h @@ -60,13 +60,12 @@ // manifest their result in the IR for passes to come. // // Attribute manifestation is not mandatory. If desired, there is support to -// generate a single LLVM-IR attribute already in the AbstractAttribute base -// class. In the simplest case, a subclass overloads -// `AbstractAttribute::getManifestPosition()` and -// `AbstractAttribute::getAttrKind()` to return the appropriate values. The -// Attributor manifestation framework will then create and place a new attribute -// if it is allowed to do so (based on the abstract state). Other use cases can -// be achieved by overloading other abstract attribute methods. +// generate a single or multiple LLVM-IR attributes already in the helper struct +// IRAttribute. In the simplest case, a subclass inherits from IRAttribute with +// a proper Attribute::AttrKind as template parameter. The Attributor +// manifestation framework will then create and place a new attribute if it is +// allowed to do so (based on the abstract state). Other use cases can be +// achieved by overloading AbstractAttribute or IRAttribute methods. // // // The "mechanics" of adding a new "abstract attribute": @@ -97,7 +96,13 @@ #ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H #define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H -#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SCCIterator.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/MustExecute.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/PassManager.h" @@ -105,6 +110,7 @@ namespace llvm { struct AbstractAttribute; struct InformationCache; +struct AAIsDead; class Function; @@ -120,6 +126,563 @@ ChangeStatus operator|(ChangeStatus l, ChangeStatus r); ChangeStatus operator&(ChangeStatus l, ChangeStatus r); ///} +/// Helper to describe and deal with positions in the LLVM-IR. +/// +/// A position in the IR is described by an anchor value and an "offset" that +/// could be the argument number, for call sites and arguments, or an indicator +/// of the "position kind". The kinds, specified in the Kind enum below, include +/// the locations in the attribute list, i.a., function scope and return value, +/// as well as a distinction between call sites and functions. Finally, there +/// are floating values that do not have a corresponding attribute list +/// position. +struct IRPosition { + virtual ~IRPosition() {} + + /// The positions we distinguish in the IR. + /// + /// The values are chosen such that the KindOrArgNo member has a value >= 1 + /// if it is an argument or call site argument while a value < 1 indicates the + /// respective kind of that value. + enum Kind : int { + IRP_INVALID = -6, ///< An invalid position. + IRP_FLOAT = -5, ///< A position that is not associated with a spot suitable + ///< for attributes. This could be any value or instruction. + IRP_RETURNED = -4, ///< An attribute for the function return value. + IRP_CALL_SITE_RETURNED = -3, ///< An attribute for a call site return value. + IRP_FUNCTION = -2, ///< An attribute for a function (scope). + IRP_CALL_SITE = -1, ///< An attribute for a call site (function scope). + IRP_ARGUMENT = 0, ///< An attribute for a function argument. + IRP_CALL_SITE_ARGUMENT = 1, ///< An attribute for a call site argument. + }; + + /// Default constructor available to create invalid positions implicitly. All + /// other positions need to be created explicitly through the appropriate + /// static member function. + IRPosition() : AnchorVal(nullptr), KindOrArgNo(IRP_INVALID) { verify(); } + + /// Create a position describing the value of \p V. + static const IRPosition value(const Value &V) { + if (auto *Arg = dyn_cast<Argument>(&V)) + return IRPosition::argument(*Arg); + if (auto *CB = dyn_cast<CallBase>(&V)) + return IRPosition::callsite_returned(*CB); + return IRPosition(const_cast<Value &>(V), IRP_FLOAT); + } + + /// Create a position describing the function scope of \p F. + static const IRPosition function(const Function &F) { + return IRPosition(const_cast<Function &>(F), IRP_FUNCTION); + } + + /// Create a position describing the returned value of \p F. + static const IRPosition returned(const Function &F) { + return IRPosition(const_cast<Function &>(F), IRP_RETURNED); + } + + /// Create a position describing the argument \p Arg. + static const IRPosition argument(const Argument &Arg) { + return IRPosition(const_cast<Argument &>(Arg), Kind(Arg.getArgNo())); + } + + /// Create a position describing the function scope of \p CB. + static const IRPosition callsite_function(const CallBase &CB) { + return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE); + } + + /// Create a position describing the returned value of \p CB. + static const IRPosition callsite_returned(const CallBase &CB) { + return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE_RETURNED); + } + + /// Create a position describing the argument of \p CB at position \p ArgNo. + static const IRPosition callsite_argument(const CallBase &CB, + unsigned ArgNo) { + return IRPosition(const_cast<CallBase &>(CB), Kind(ArgNo)); + } + + /// Create a position describing the function scope of \p ICS. + static const IRPosition callsite_function(ImmutableCallSite ICS) { + return IRPosition::callsite_function(cast<CallBase>(*ICS.getInstruction())); + } + + /// Create a position describing the returned value of \p ICS. + static const IRPosition callsite_returned(ImmutableCallSite ICS) { + return IRPosition::callsite_returned(cast<CallBase>(*ICS.getInstruction())); + } + + /// Create a position describing the argument of \p ICS at position \p ArgNo. + static const IRPosition callsite_argument(ImmutableCallSite ICS, + unsigned ArgNo) { + return IRPosition::callsite_argument(cast<CallBase>(*ICS.getInstruction()), + ArgNo); + } + + /// Create a position describing the argument of \p ACS at position \p ArgNo. + static const IRPosition callsite_argument(AbstractCallSite ACS, + unsigned ArgNo) { + int CSArgNo = ACS.getCallArgOperandNo(ArgNo); + if (CSArgNo >= 0) + return IRPosition::callsite_argument( + cast<CallBase>(*ACS.getInstruction()), CSArgNo); + return IRPosition(); + } + + /// Create a position with function scope matching the "context" of \p IRP. + /// If \p IRP is a call site (see isAnyCallSitePosition()) then the result + /// will be a call site position, otherwise the function position of the + /// associated function. + static const IRPosition function_scope(const IRPosition &IRP) { + if (IRP.isAnyCallSitePosition()) { + return IRPosition::callsite_function( + cast<CallBase>(IRP.getAnchorValue())); + } + assert(IRP.getAssociatedFunction()); + return IRPosition::function(*IRP.getAssociatedFunction()); + } + + bool operator==(const IRPosition &RHS) const { + return (AnchorVal == RHS.AnchorVal) && (KindOrArgNo == RHS.KindOrArgNo); + } + bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); } + + /// Return the value this abstract attribute is anchored with. + /// + /// The anchor value might not be the associated value if the latter is not + /// sufficient to determine where arguments will be manifested. This is, so + /// far, only the case for call site arguments as the value is not sufficient + /// to pinpoint them. Instead, we can use the call site as an anchor. + /// + ///{ + Value &getAnchorValue() { + assert(KindOrArgNo != IRP_INVALID && + "Invalid position does not have an anchor value!"); + return *AnchorVal; + } + const Value &getAnchorValue() const { + return const_cast<IRPosition *>(this)->getAnchorValue(); + } + ///} + + /// Return the associated function, if any. + /// + ///{ + Function *getAssociatedFunction() { + if (auto *CB = dyn_cast<CallBase>(AnchorVal)) + return CB->getCalledFunction(); + assert(KindOrArgNo != IRP_INVALID && + "Invalid position does not have an anchor scope!"); + Value &V = getAnchorValue(); + if (isa<Function>(V)) + return &cast<Function>(V); + if (isa<Argument>(V)) + return cast<Argument>(V).getParent(); + if (isa<Instruction>(V)) + return cast<Instruction>(V).getFunction(); + return nullptr; + } + const Function *getAssociatedFunction() const { + return const_cast<IRPosition *>(this)->getAssociatedFunction(); + } + ///} + + /// Return the associated argument, if any. + /// + ///{ + Argument *getAssociatedArgument() { + if (auto *Arg = dyn_cast<Argument>(&getAnchorValue())) + return Arg; + int ArgNo = getArgNo(); + if (ArgNo < 0) + return nullptr; + Function *AssociatedFn = getAssociatedFunction(); + if (!AssociatedFn || AssociatedFn->arg_size() <= unsigned(ArgNo)) + return nullptr; + return AssociatedFn->arg_begin() + ArgNo; + } + const Argument *getAssociatedArgument() const { + return const_cast<IRPosition *>(this)->getAssociatedArgument(); + } + ///} + + /// Return true if the position refers to a function interface, that is the + /// function scope, the function return, or an argumnt. + bool isFnInterfaceKind() const { + switch (getPositionKind()) { + case IRPosition::IRP_FUNCTION: + case IRPosition::IRP_RETURNED: + case IRPosition::IRP_ARGUMENT: + return true; + default: + return false; + } + } + + /// Return the Function surrounding the anchor value. + /// + ///{ + Function *getAnchorScope() { + Value &V = getAnchorValue(); + if (isa<Function>(V)) + return &cast<Function>(V); + if (isa<Argument>(V)) + return cast<Argument>(V).getParent(); + if (isa<Instruction>(V)) + return cast<Instruction>(V).getFunction(); + return nullptr; + } + const Function *getAnchorScope() const { + return const_cast<IRPosition *>(this)->getAnchorScope(); + } + ///} + + /// Return the context instruction, if any. + /// + ///{ + Instruction *getCtxI() { + Value &V = getAnchorValue(); + if (auto *I = dyn_cast<Instruction>(&V)) + return I; + if (auto *Arg = dyn_cast<Argument>(&V)) + if (!Arg->getParent()->isDeclaration()) + return &Arg->getParent()->getEntryBlock().front(); + if (auto *F = dyn_cast<Function>(&V)) + if (!F->isDeclaration()) + return &(F->getEntryBlock().front()); + return nullptr; + } + const Instruction *getCtxI() const { + return const_cast<IRPosition *>(this)->getCtxI(); + } + ///} + + /// Return the value this abstract attribute is associated with. + /// + ///{ + Value &getAssociatedValue() { + assert(KindOrArgNo != IRP_INVALID && + "Invalid position does not have an associated value!"); + if (getArgNo() < 0 || isa<Argument>(AnchorVal)) + return *AnchorVal; + assert(isa<CallBase>(AnchorVal) && "Expected a call base!"); + return *cast<CallBase>(AnchorVal)->getArgOperand(getArgNo()); + } + const Value &getAssociatedValue() const { + return const_cast<IRPosition *>(this)->getAssociatedValue(); + } + ///} + + /// Return the argument number of the associated value if it is an argument or + /// call site argument, otherwise a negative value. + int getArgNo() const { return KindOrArgNo; } + + /// Return the index in the attribute list for this position. + unsigned getAttrIdx() const { + switch (getPositionKind()) { + case IRPosition::IRP_INVALID: + case IRPosition::IRP_FLOAT: + break; + case IRPosition::IRP_FUNCTION: + case IRPosition::IRP_CALL_SITE: + return AttributeList::FunctionIndex; + case IRPosition::IRP_RETURNED: + case IRPosition::IRP_CALL_SITE_RETURNED: + return AttributeList::ReturnIndex; + case IRPosition::IRP_ARGUMENT: + case IRPosition::IRP_CALL_SITE_ARGUMENT: + return KindOrArgNo + AttributeList::FirstArgIndex; + } + llvm_unreachable( + "There is no attribute index for a floating or invalid position!"); + } + + /// Return the associated position kind. + Kind getPositionKind() const { + if (getArgNo() >= 0) { + assert(((isa<Argument>(getAnchorValue()) && + isa<Argument>(getAssociatedValue())) || + isa<CallBase>(getAnchorValue())) && + "Expected argument or call base due to argument number!"); + if (isa<CallBase>(getAnchorValue())) + return IRP_CALL_SITE_ARGUMENT; + return IRP_ARGUMENT; + } + + assert(KindOrArgNo < 0 && + "Expected (call site) arguments to never reach this point!"); + return Kind(KindOrArgNo); + } + + /// TODO: Figure out if the attribute related helper functions should live + /// here or somewhere else. + + /// Return true if any kind in \p AKs existing in the IR at a position that + /// will affect this one. See also getAttrs(...). + /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions, + /// e.g., the function position if this is an + /// argument position, should be ignored. + bool hasAttr(ArrayRef<Attribute::AttrKind> AKs, + bool IgnoreSubsumingPositions = false) const; + + /// Return the attributes of any kind in \p AKs existing in the IR at a + /// position that will affect this one. While each position can only have a + /// single attribute of any kind in \p AKs, there are "subsuming" positions + /// that could have an attribute as well. This method returns all attributes + /// found in \p Attrs. + void getAttrs(ArrayRef<Attribute::AttrKind> AKs, + SmallVectorImpl<Attribute> &Attrs) const; + + /// Return the attribute of kind \p AK existing in the IR at this position. + Attribute getAttr(Attribute::AttrKind AK) const { + if (getPositionKind() == IRP_INVALID || getPositionKind() == IRP_FLOAT) + return Attribute(); + + AttributeList AttrList; + if (ImmutableCallSite ICS = ImmutableCallSite(&getAnchorValue())) + AttrList = ICS.getAttributes(); + else + AttrList = getAssociatedFunction()->getAttributes(); + + if (AttrList.hasAttribute(getAttrIdx(), AK)) + return AttrList.getAttribute(getAttrIdx(), AK); + return Attribute(); + } + + /// Remove the attribute of kind \p AKs existing in the IR at this position. + void removeAttrs(ArrayRef<Attribute::AttrKind> AKs) { + if (getPositionKind() == IRP_INVALID || getPositionKind() == IRP_FLOAT) + return; + + AttributeList AttrList; + CallSite CS = CallSite(&getAnchorValue()); + if (CS) + AttrList = CS.getAttributes(); + else + AttrList = getAssociatedFunction()->getAttributes(); + + LLVMContext &Ctx = getAnchorValue().getContext(); + for (Attribute::AttrKind AK : AKs) + AttrList = AttrList.removeAttribute(Ctx, getAttrIdx(), AK); + + if (CS) + CS.setAttributes(AttrList); + else + getAssociatedFunction()->setAttributes(AttrList); + } + + bool isAnyCallSitePosition() const { + switch (getPositionKind()) { + case IRPosition::IRP_CALL_SITE: + case IRPosition::IRP_CALL_SITE_RETURNED: + case IRPosition::IRP_CALL_SITE_ARGUMENT: + return true; + default: + return false; + } + } + + /// Special DenseMap key values. + /// + ///{ + static const IRPosition EmptyKey; + static const IRPosition TombstoneKey; + ///} + +private: + /// Private constructor for special values only! + explicit IRPosition(int KindOrArgNo) + : AnchorVal(0), KindOrArgNo(KindOrArgNo) {} + + /// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK. + explicit IRPosition(Value &AnchorVal, Kind PK) + : AnchorVal(&AnchorVal), KindOrArgNo(PK) { + verify(); + } + + /// Verify internal invariants. + void verify(); + + /// The value this position is anchored at. + Value *AnchorVal; + + /// The argument number, if non-negative, or the position "kind". + int KindOrArgNo; +}; + +/// Helper that allows IRPosition as a key in a DenseMap. +template <> struct DenseMapInfo<IRPosition> { + static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; } + static inline IRPosition getTombstoneKey() { + return IRPosition::TombstoneKey; + } + static unsigned getHashValue(const IRPosition &IRP) { + return (DenseMapInfo<Value *>::getHashValue(&IRP.getAnchorValue()) << 4) ^ + (unsigned(IRP.getArgNo())); + } + static bool isEqual(const IRPosition &LHS, const IRPosition &RHS) { + return LHS == RHS; + } +}; + +/// A visitor class for IR positions. +/// +/// Given a position P, the SubsumingPositionIterator allows to visit "subsuming +/// positions" wrt. attributes/information. Thus, if a piece of information +/// holds for a subsuming position, it also holds for the position P. +/// +/// The subsuming positions always include the initial position and then, +/// depending on the position kind, additionally the following ones: +/// - for IRP_RETURNED: +/// - the function (IRP_FUNCTION) +/// - for IRP_ARGUMENT: +/// - the function (IRP_FUNCTION) +/// - for IRP_CALL_SITE: +/// - the callee (IRP_FUNCTION), if known +/// - for IRP_CALL_SITE_RETURNED: +/// - the callee (IRP_RETURNED), if known +/// - the call site (IRP_FUNCTION) +/// - the callee (IRP_FUNCTION), if known +/// - for IRP_CALL_SITE_ARGUMENT: +/// - the argument of the callee (IRP_ARGUMENT), if known +/// - the callee (IRP_FUNCTION), if known +/// - the position the call site argument is associated with if it is not +/// anchored to the call site, e.g., if it is an arugment then the argument +/// (IRP_ARGUMENT) +class SubsumingPositionIterator { + SmallVector<IRPosition, 4> IRPositions; + using iterator = decltype(IRPositions)::iterator; + +public: + SubsumingPositionIterator(const IRPosition &IRP); + iterator begin() { return IRPositions.begin(); } + iterator end() { return IRPositions.end(); } +}; + +/// Wrapper for FunctoinAnalysisManager. +struct AnalysisGetter { + template <typename Analysis> + typename Analysis::Result *getAnalysis(const Function &F) { + if (!MAM || !F.getParent()) + return nullptr; + auto &FAM = MAM->getResult<FunctionAnalysisManagerModuleProxy>( + const_cast<Module &>(*F.getParent())) + .getManager(); + return &FAM.getResult<Analysis>(const_cast<Function &>(F)); + } + + template <typename Analysis> + typename Analysis::Result *getAnalysis(const Module &M) { + if (!MAM) + return nullptr; + return &MAM->getResult<Analysis>(const_cast<Module &>(M)); + } + AnalysisGetter(ModuleAnalysisManager &MAM) : MAM(&MAM) {} + AnalysisGetter() {} + +private: + ModuleAnalysisManager *MAM = nullptr; +}; + +/// Data structure to hold cached (LLVM-IR) information. +/// +/// All attributes are given an InformationCache object at creation time to +/// avoid inspection of the IR by all of them individually. This default +/// InformationCache will hold information required by 'default' attributes, +/// thus the ones deduced when Attributor::identifyDefaultAbstractAttributes(..) +/// is called. +/// +/// If custom abstract attributes, registered manually through +/// Attributor::registerAA(...), need more information, especially if it is not +/// reusable, it is advised to inherit from the InformationCache and cast the +/// instance down in the abstract attributes. +struct InformationCache { + InformationCache(const Module &M, AnalysisGetter &AG) + : DL(M.getDataLayout()), Explorer(/* ExploreInterBlock */ true), AG(AG) { + + CallGraph *CG = AG.getAnalysis<CallGraphAnalysis>(M); + if (!CG) + return; + + DenseMap<const Function *, unsigned> SccSize; + for (scc_iterator<CallGraph *> I = scc_begin(CG); !I.isAtEnd(); ++I) { + for (CallGraphNode *Node : *I) + SccSize[Node->getFunction()] = I->size(); + } + SccSizeOpt = std::move(SccSize); + } + + /// A map type from opcodes to instructions with this opcode. + using OpcodeInstMapTy = DenseMap<unsigned, SmallVector<Instruction *, 32>>; + + /// Return the map that relates "interesting" opcodes with all instructions + /// with that opcode in \p F. + OpcodeInstMapTy &getOpcodeInstMapForFunction(const Function &F) { + return FuncInstOpcodeMap[&F]; + } + + /// A vector type to hold instructions. + using InstructionVectorTy = std::vector<Instruction *>; + + /// Return the instructions in \p F that may read or write memory. + InstructionVectorTy &getReadOrWriteInstsForFunction(const Function &F) { + return FuncRWInstsMap[&F]; + } + + /// Return MustBeExecutedContextExplorer + MustBeExecutedContextExplorer &getMustBeExecutedContextExplorer() { + return Explorer; + } + + /// Return TargetLibraryInfo for function \p F. + TargetLibraryInfo *getTargetLibraryInfoForFunction(const Function &F) { + return AG.getAnalysis<TargetLibraryAnalysis>(F); + } + + /// Return AliasAnalysis Result for function \p F. + AAResults *getAAResultsForFunction(const Function &F) { + return AG.getAnalysis<AAManager>(F); + } + + /// Return SCC size on call graph for function \p F. + unsigned getSccSize(const Function &F) { + if (!SccSizeOpt.hasValue()) + return 0; + return (SccSizeOpt.getValue())[&F]; + } + + /// Return datalayout used in the module. + const DataLayout &getDL() { return DL; } + +private: + /// A map type from functions to opcode to instruction maps. + using FuncInstOpcodeMapTy = DenseMap<const Function *, OpcodeInstMapTy>; + + /// A map type from functions to their read or write instructions. + using FuncRWInstsMapTy = DenseMap<const Function *, InstructionVectorTy>; + + /// A nested map that remembers all instructions in a function with a certain + /// instruction opcode (Instruction::getOpcode()). + FuncInstOpcodeMapTy FuncInstOpcodeMap; + + /// A map from functions to their instructions that may read or write memory. + FuncRWInstsMapTy FuncRWInstsMap; + + /// The datalayout used in the module. + const DataLayout &DL; + + /// MustBeExecutedContextExplorer + MustBeExecutedContextExplorer Explorer; + + /// Getters for analysis. + AnalysisGetter &AG; + + /// Cache result for scc size in the call graph + Optional<DenseMap<const Function *, unsigned>> SccSizeOpt; + + /// Give the Attributor access to the members so + /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them. + friend struct Attributor; +}; + /// The fixpoint analysis framework that orchestrates the attribute deduction. /// /// The Attributor provides a general abstract analysis framework (guided @@ -148,6 +711,18 @@ ChangeStatus operator&(ChangeStatus l, ChangeStatus r); /// NOTE: The mechanics of adding a new "concrete" abstract attribute are /// described in the file comment. struct Attributor { + /// Constructor + /// + /// \param InfoCache Cache to hold various information accessible for + /// the abstract attributes. + /// \param DepRecomputeInterval Number of iterations until the dependences + /// between abstract attributes are recomputed. + /// \param Whitelist If not null, a set limiting the attribute opportunities. + Attributor(InformationCache &InfoCache, unsigned DepRecomputeInterval, + DenseSet<const char *> *Whitelist = nullptr) + : InfoCache(InfoCache), DepRecomputeInterval(DepRecomputeInterval), + Whitelist(Whitelist) {} + ~Attributor() { DeleteContainerPointers(AllAbstractAttributes); } /// Run the analyses until a fixpoint is reached or enforced (timeout). @@ -156,12 +731,13 @@ struct Attributor { /// as the Attributor is not destroyed (it owns the attributes now). /// /// \Returns CHANGED if the IR was changed, otherwise UNCHANGED. - ChangeStatus run(); + ChangeStatus run(Module &M); - /// Lookup an abstract attribute of type \p AAType anchored at value \p V and - /// argument number \p ArgNo. If no attribute is found and \p V is a call base - /// instruction, the called function is tried as a value next. Thus, the - /// returned abstract attribute might be anchored at the callee of \p V. + /// Lookup an abstract attribute of type \p AAType at position \p IRP. While + /// no abstract attribute is found equivalent positions are checked, see + /// SubsumingPositionIterator. Thus, the returned abstract attribute + /// might be anchored at a different position, e.g., the callee if \p IRP is a + /// call base. /// /// This method is the only (supported) way an abstract attribute can retrieve /// information from another abstract attribute. As an example, take an @@ -170,51 +746,29 @@ struct Attributor { /// most optimistic information for other abstract attributes in-flight, e.g. /// the one reasoning about the "captured" state for the argument or the one /// reasoning on the memory access behavior of the function as a whole. + /// + /// If the flag \p TrackDependence is set to false the dependence from + /// \p QueryingAA to the return abstract attribute is not automatically + /// recorded. This should only be used if the caller will record the + /// dependence explicitly if necessary, thus if it the returned abstract + /// attribute is used for reasoning. To record the dependences explicitly use + /// the `Attributor::recordDependence` method. template <typename AAType> - const AAType *getAAFor(AbstractAttribute &QueryingAA, const Value &V, - int ArgNo = -1) { - static_assert(std::is_base_of<AbstractAttribute, AAType>::value, - "Cannot query an attribute with a type not derived from " - "'AbstractAttribute'!"); - assert(AAType::ID != Attribute::None && - "Cannot lookup generic abstract attributes!"); - - // Determine the argument number automatically for llvm::Arguments if none - // is set. Do not override a given one as it could be a use of the argument - // in a call site. - if (ArgNo == -1) - if (auto *Arg = dyn_cast<Argument>(&V)) - ArgNo = Arg->getArgNo(); - - // If a function was given together with an argument number, perform the - // lookup for the actual argument instead. Don't do it for variadic - // arguments. - if (ArgNo >= 0 && isa<Function>(&V) && - cast<Function>(&V)->arg_size() > (size_t)ArgNo) - return getAAFor<AAType>( - QueryingAA, *(cast<Function>(&V)->arg_begin() + ArgNo), ArgNo); - - // Lookup the abstract attribute of type AAType. If found, return it after - // registering a dependence of QueryingAA on the one returned attribute. - const auto &KindToAbstractAttributeMap = AAMap.lookup({&V, ArgNo}); - if (AAType *AA = static_cast<AAType *>( - KindToAbstractAttributeMap.lookup(AAType::ID))) { - // Do not return an attribute with an invalid state. This minimizes checks - // at the calls sites and allows the fallback below to kick in. - if (AA->getState().isValidState()) { - QueryMap[AA].insert(&QueryingAA); - return AA; - } - } - - // If no abstract attribute was found and we look for a call site argument, - // defer to the actual argument instead. - ImmutableCallSite ICS(&V); - if (ICS && ICS.getCalledValue()) - return getAAFor<AAType>(QueryingAA, *ICS.getCalledValue(), ArgNo); + const AAType &getAAFor(const AbstractAttribute &QueryingAA, + const IRPosition &IRP, bool TrackDependence = true) { + return getOrCreateAAFor<AAType>(IRP, &QueryingAA, TrackDependence); + } - // No matching attribute found - return nullptr; + /// Explicitly record a dependence from \p FromAA to \p ToAA, that is if + /// \p FromAA changes \p ToAA should be updated as well. + /// + /// This method should be used in conjunction with the `getAAFor` method and + /// with the TrackDependence flag passed to the method set to false. This can + /// be beneficial to avoid false dependences but it requires the users of + /// `getAAFor` to explicitly record true dependences through this method. + void recordDependence(const AbstractAttribute &FromAA, + const AbstractAttribute &ToAA) { + QueryMap[&FromAA].insert(const_cast<AbstractAttribute *>(&ToAA)); } /// Introduce a new abstract attribute into the fixpoint analysis. @@ -222,126 +776,242 @@ struct Attributor { /// Note that ownership of the attribute is given to the Attributor. It will /// invoke delete for the Attributor on destruction of the Attributor. /// - /// Attributes are identified by - /// (1) their anchored value (see AA.getAnchoredValue()), - /// (2) their argument number (\p ArgNo, or Argument::getArgNo()), and - /// (3) their default attribute kind (see AAType::ID). - template <typename AAType> AAType ®isterAA(AAType &AA, int ArgNo = -1) { + /// Attributes are identified by their IR position (AAType::getIRPosition()) + /// and the address of their static member (see AAType::ID). + template <typename AAType> AAType ®isterAA(AAType &AA) { static_assert(std::is_base_of<AbstractAttribute, AAType>::value, "Cannot register an attribute with a type not derived from " "'AbstractAttribute'!"); - - // Determine the anchor value and the argument number which are used to - // lookup the attribute together with AAType::ID. If passed an argument, - // use its argument number but do not override a given one as it could be a - // use of the argument at a call site. - Value &AnchoredVal = AA.getAnchoredValue(); - if (ArgNo == -1) - if (auto *Arg = dyn_cast<Argument>(&AnchoredVal)) - ArgNo = Arg->getArgNo(); - // Put the attribute in the lookup map structure and the container we use to // keep track of all attributes. - AAMap[{&AnchoredVal, ArgNo}][AAType::ID] = &AA; + IRPosition &IRP = AA.getIRPosition(); + auto &KindToAbstractAttributeMap = AAMap[IRP]; + assert(!KindToAbstractAttributeMap.count(&AAType::ID) && + "Attribute already in map!"); + KindToAbstractAttributeMap[&AAType::ID] = &AA; AllAbstractAttributes.push_back(&AA); return AA; } + /// Return the internal information cache. + InformationCache &getInfoCache() { return InfoCache; } + /// Determine opportunities to derive 'default' attributes in \p F and create /// abstract attribute objects for them. /// /// \param F The function that is checked for attribute opportunities. - /// \param InfoCache A cache for information queryable by the new attributes. - /// \param Whitelist If not null, a set limiting the attribute opportunities. /// /// Note that abstract attribute instances are generally created even if the /// IR already contains the information they would deduce. The most important /// reason for this is the single interface, the one of the abstract attribute /// instance, which can be queried without the need to look at the IR in /// various places. - void identifyDefaultAbstractAttributes( - Function &F, InformationCache &InfoCache, - DenseSet</* Attribute::AttrKind */ unsigned> *Whitelist = nullptr); + void identifyDefaultAbstractAttributes(Function &F); + + /// Initialize the information cache for queries regarding function \p F. + /// + /// This method needs to be called for all function that might be looked at + /// through the information cache interface *prior* to looking at them. + void initializeInformationCache(Function &F); + + /// Mark the internal function \p F as live. + /// + /// This will trigger the identification and initialization of attributes for + /// \p F. + void markLiveInternalFunction(const Function &F) { + assert(F.hasLocalLinkage() && + "Only local linkage is assumed dead initially."); + + identifyDefaultAbstractAttributes(const_cast<Function &>(F)); + } + + /// Record that \p I is deleted after information was manifested. + void deleteAfterManifest(Instruction &I) { ToBeDeletedInsts.insert(&I); } + + /// Record that \p BB is deleted after information was manifested. + void deleteAfterManifest(BasicBlock &BB) { ToBeDeletedBlocks.insert(&BB); } + + /// Record that \p F is deleted after information was manifested. + void deleteAfterManifest(Function &F) { ToBeDeletedFunctions.insert(&F); } + + /// Return true if \p AA (or its context instruction) is assumed dead. + /// + /// If \p LivenessAA is not provided it is queried. + bool isAssumedDead(const AbstractAttribute &AA, const AAIsDead *LivenessAA); /// Check \p Pred on all function call sites. /// /// This method will evaluate \p Pred on call sites and return /// true if \p Pred holds in every call sites. However, this is only possible /// all call sites are known, hence the function has internal linkage. - bool checkForAllCallSites(Function &F, std::function<bool(CallSite)> &Pred, + bool checkForAllCallSites(const function_ref<bool(AbstractCallSite)> &Pred, + const AbstractAttribute &QueryingAA, bool RequireAllCallSites); + /// Check \p Pred on all values potentially returned by \p F. + /// + /// This method will evaluate \p Pred on all values potentially returned by + /// the function associated with \p QueryingAA. The returned values are + /// matched with their respective return instructions. Returns true if \p Pred + /// holds on all of them. + bool checkForAllReturnedValuesAndReturnInsts( + const function_ref<bool(Value &, const SmallSetVector<ReturnInst *, 4> &)> + &Pred, + const AbstractAttribute &QueryingAA); + + /// Check \p Pred on all values potentially returned by the function + /// associated with \p QueryingAA. + /// + /// This is the context insensitive version of the method above. + bool checkForAllReturnedValues(const function_ref<bool(Value &)> &Pred, + const AbstractAttribute &QueryingAA); + + /// Check \p Pred on all instructions with an opcode present in \p Opcodes. + /// + /// This method will evaluate \p Pred on all instructions with an opcode + /// present in \p Opcode and return true if \p Pred holds on all of them. + bool checkForAllInstructions(const function_ref<bool(Instruction &)> &Pred, + const AbstractAttribute &QueryingAA, + const ArrayRef<unsigned> &Opcodes); + + /// Check \p Pred on all call-like instructions (=CallBased derived). + /// + /// See checkForAllCallLikeInstructions(...) for more information. + bool + checkForAllCallLikeInstructions(const function_ref<bool(Instruction &)> &Pred, + const AbstractAttribute &QueryingAA) { + return checkForAllInstructions(Pred, QueryingAA, + {(unsigned)Instruction::Invoke, + (unsigned)Instruction::CallBr, + (unsigned)Instruction::Call}); + } + + /// Check \p Pred on all Read/Write instructions. + /// + /// This method will evaluate \p Pred on all instructions that read or write + /// to memory present in the information cache and return true if \p Pred + /// holds on all of them. + bool checkForAllReadWriteInstructions( + const llvm::function_ref<bool(Instruction &)> &Pred, + AbstractAttribute &QueryingAA); + + /// Return the data layout associated with the anchor scope. + const DataLayout &getDataLayout() const { return InfoCache.DL; } + private: + /// Check \p Pred on all call sites of \p Fn. + /// + /// This method will evaluate \p Pred on call sites and return + /// true if \p Pred holds in every call sites. However, this is only possible + /// all call sites are known, hence the function has internal linkage. + bool checkForAllCallSites(const function_ref<bool(AbstractCallSite)> &Pred, + const Function &Fn, bool RequireAllCallSites, + const AbstractAttribute *QueryingAA); + + /// The private version of getAAFor that allows to omit a querying abstract + /// attribute. See also the public getAAFor method. + template <typename AAType> + const AAType &getOrCreateAAFor(const IRPosition &IRP, + const AbstractAttribute *QueryingAA = nullptr, + bool TrackDependence = false) { + if (const AAType *AAPtr = + lookupAAFor<AAType>(IRP, QueryingAA, TrackDependence)) + return *AAPtr; + + // No matching attribute found, create one. + // Use the static create method. + auto &AA = AAType::createForPosition(IRP, *this); + registerAA(AA); + + // For now we ignore naked and optnone functions. + bool Invalidate = Whitelist && !Whitelist->count(&AAType::ID); + if (const Function *Fn = IRP.getAnchorScope()) + Invalidate |= Fn->hasFnAttribute(Attribute::Naked) || + Fn->hasFnAttribute(Attribute::OptimizeNone); + + // Bootstrap the new attribute with an initial update to propagate + // information, e.g., function -> call site. If it is not on a given + // whitelist we will not perform updates at all. + if (Invalidate) { + AA.getState().indicatePessimisticFixpoint(); + return AA; + } + + AA.initialize(*this); + AA.update(*this); + + if (TrackDependence && AA.getState().isValidState()) + QueryMap[&AA].insert(const_cast<AbstractAttribute *>(QueryingAA)); + return AA; + } + + /// Return the attribute of \p AAType for \p IRP if existing. + template <typename AAType> + const AAType *lookupAAFor(const IRPosition &IRP, + const AbstractAttribute *QueryingAA = nullptr, + bool TrackDependence = false) { + static_assert(std::is_base_of<AbstractAttribute, AAType>::value, + "Cannot query an attribute with a type not derived from " + "'AbstractAttribute'!"); + assert((QueryingAA || !TrackDependence) && + "Cannot track dependences without a QueryingAA!"); + + // Lookup the abstract attribute of type AAType. If found, return it after + // registering a dependence of QueryingAA on the one returned attribute. + const auto &KindToAbstractAttributeMap = AAMap.lookup(IRP); + if (AAType *AA = static_cast<AAType *>( + KindToAbstractAttributeMap.lookup(&AAType::ID))) { + // Do not register a dependence on an attribute with an invalid state. + if (TrackDependence && AA->getState().isValidState()) + QueryMap[AA].insert(const_cast<AbstractAttribute *>(QueryingAA)); + return AA; + } + return nullptr; + } + /// The set of all abstract attributes. ///{ using AAVector = SmallVector<AbstractAttribute *, 64>; AAVector AllAbstractAttributes; ///} - /// A nested map to lookup abstract attributes based on the anchored value and - /// an argument positions (or -1) on the outer level, and attribute kinds - /// (Attribute::AttrKind) on the inner level. + /// A nested map to lookup abstract attributes based on the argument position + /// on the outer level, and the addresses of the static member (AAType::ID) on + /// the inner level. ///{ - using KindToAbstractAttributeMap = DenseMap<unsigned, AbstractAttribute *>; - DenseMap<std::pair<const Value *, int>, KindToAbstractAttributeMap> AAMap; + using KindToAbstractAttributeMap = + DenseMap<const char *, AbstractAttribute *>; + DenseMap<IRPosition, KindToAbstractAttributeMap> AAMap; ///} /// A map from abstract attributes to the ones that queried them through calls /// to the getAAFor<...>(...) method. ///{ using QueryMapTy = - DenseMap<AbstractAttribute *, SetVector<AbstractAttribute *>>; + MapVector<const AbstractAttribute *, SetVector<AbstractAttribute *>>; QueryMapTy QueryMap; ///} -}; - -/// Data structure to hold cached (LLVM-IR) information. -/// -/// All attributes are given an InformationCache object at creation time to -/// avoid inspection of the IR by all of them individually. This default -/// InformationCache will hold information required by 'default' attributes, -/// thus the ones deduced when Attributor::identifyDefaultAbstractAttributes(..) -/// is called. -/// -/// If custom abstract attributes, registered manually through -/// Attributor::registerAA(...), need more information, especially if it is not -/// reusable, it is advised to inherit from the InformationCache and cast the -/// instance down in the abstract attributes. -struct InformationCache { - /// A map type from opcodes to instructions with this opcode. - using OpcodeInstMapTy = DenseMap<unsigned, SmallVector<Instruction *, 32>>; - - /// Return the map that relates "interesting" opcodes with all instructions - /// with that opcode in \p F. - OpcodeInstMapTy &getOpcodeInstMapForFunction(Function &F) { - return FuncInstOpcodeMap[&F]; - } - /// A vector type to hold instructions. - using InstructionVectorTy = std::vector<Instruction *>; - - /// Return the instructions in \p F that may read or write memory. - InstructionVectorTy &getReadOrWriteInstsForFunction(Function &F) { - return FuncRWInstsMap[&F]; - } - -private: - /// A map type from functions to opcode to instruction maps. - using FuncInstOpcodeMapTy = DenseMap<Function *, OpcodeInstMapTy>; + /// The information cache that holds pre-processed (LLVM-IR) information. + InformationCache &InfoCache; - /// A map type from functions to their read or write instructions. - using FuncRWInstsMapTy = DenseMap<Function *, InstructionVectorTy>; + /// Number of iterations until the dependences between abstract attributes are + /// recomputed. + const unsigned DepRecomputeInterval; - /// A nested map that remembers all instructions in a function with a certain - /// instruction opcode (Instruction::getOpcode()). - FuncInstOpcodeMapTy FuncInstOpcodeMap; + /// If not null, a set limiting the attribute opportunities. + const DenseSet<const char *> *Whitelist; - /// A map from functions to their instructions that may read or write memory. - FuncRWInstsMapTy FuncRWInstsMap; + /// A set to remember the functions we already assume to be live and visited. + DenseSet<const Function *> VisitedFunctions; - /// Give the Attributor access to the members so - /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them. - friend struct Attributor; + /// Functions, blocks, and instructions we delete after manifest is done. + /// + ///{ + SmallPtrSet<Function *, 8> ToBeDeletedFunctions; + SmallPtrSet<BasicBlock *, 8> ToBeDeletedBlocks; + SmallPtrSet<Instruction *, 8> ToBeDeletedInsts; + ///} }; /// An interface to query the internal state of an abstract attribute. @@ -375,13 +1045,17 @@ struct AbstractState { /// /// This will usually make the optimistically assumed state the known to be /// true state. - virtual void indicateOptimisticFixpoint() = 0; + /// + /// \returns ChangeStatus::UNCHANGED as the assumed value should not change. + virtual ChangeStatus indicateOptimisticFixpoint() = 0; /// Indicate that the abstract state should converge to the pessimistic state. /// /// This will usually revert the optimistically assumed state to the known to /// be true state. - virtual void indicatePessimisticFixpoint() = 0; + /// + /// \returns ChangeStatus::CHANGED as the assumed value may change. + virtual ChangeStatus indicatePessimisticFixpoint() = 0; }; /// Simple state with integers encoding. @@ -412,10 +1086,16 @@ struct IntegerState : public AbstractState { bool isAtFixpoint() const override { return Assumed == Known; } /// See AbstractState::indicateOptimisticFixpoint(...) - void indicateOptimisticFixpoint() override { Known = Assumed; } + ChangeStatus indicateOptimisticFixpoint() override { + Known = Assumed; + return ChangeStatus::UNCHANGED; + } /// See AbstractState::indicatePessimisticFixpoint(...) - void indicatePessimisticFixpoint() override { Assumed = Known; } + ChangeStatus indicatePessimisticFixpoint() override { + Assumed = Known; + return ChangeStatus::CHANGED; + } /// Return the known state encoding base_t getKnown() const { return Known; } @@ -448,6 +1128,12 @@ struct IntegerState : public AbstractState { return *this; } + /// Remove the bits in \p BitsEncoding from the "known bits". + IntegerState &removeKnownBits(base_t BitsEncoding) { + Known = (Known & ~BitsEncoding); + return *this; + } + /// Keep only "assumed bits" also set in \p BitsEncoding but all known ones. IntegerState &intersectAssumedBits(base_t BitsEncoding) { // Make sure we never loose any "known bits". @@ -455,6 +1141,62 @@ struct IntegerState : public AbstractState { return *this; } + /// Take minimum of assumed and \p Value. + IntegerState &takeAssumedMinimum(base_t Value) { + // Make sure we never loose "known value". + Assumed = std::max(std::min(Assumed, Value), Known); + return *this; + } + + /// Take maximum of known and \p Value. + IntegerState &takeKnownMaximum(base_t Value) { + // Make sure we never loose "known value". + Assumed = std::max(Value, Assumed); + Known = std::max(Value, Known); + return *this; + } + + /// Equality for IntegerState. + bool operator==(const IntegerState &R) const { + return this->getAssumed() == R.getAssumed() && + this->getKnown() == R.getKnown(); + } + + /// Inequality for IntegerState. + bool operator!=(const IntegerState &R) const { return !(*this == R); } + + /// "Clamp" this state with \p R. The result is the minimum of the assumed + /// information but not less than what was known before. + /// + /// TODO: Consider replacing the operator with a call or using it only when + /// we can also take the maximum of the known information, thus when + /// \p R is not dependent on additional assumed state. + IntegerState operator^=(const IntegerState &R) { + takeAssumedMinimum(R.Assumed); + return *this; + } + + /// "Clamp" this state with \p R. The result is the maximum of the known + /// information but not more than what was assumed before. + IntegerState operator+=(const IntegerState &R) { + takeKnownMaximum(R.Known); + return *this; + } + + /// Make this the minimum, known and assumed, of this state and \p R. + IntegerState operator&=(const IntegerState &R) { + Known = std::min(Known, R.Known); + Assumed = std::min(Assumed, R.Assumed); + return *this; + } + + /// Make this the maximum, known and assumed, of this state and \p R. + IntegerState operator|=(const IntegerState &R) { + Known = std::max(Known, R.Known); + Assumed = std::max(Assumed, R.Assumed); + return *this; + } + private: /// The known state encoding in an integer of type base_t. base_t Known = getWorstState(); @@ -468,6 +1210,77 @@ struct BooleanState : public IntegerState { BooleanState() : IntegerState(1){}; }; +/// Helper struct necessary as the modular build fails if the virtual method +/// IRAttribute::manifest is defined in the Attributor.cpp. +struct IRAttributeManifest { + static ChangeStatus manifestAttrs(Attributor &A, IRPosition &IRP, + const ArrayRef<Attribute> &DeducedAttrs); +}; + +/// Helper to tie a abstract state implementation to an abstract attribute. +template <typename StateTy, typename Base> +struct StateWrapper : public StateTy, public Base { + /// Provide static access to the type of the state. + using StateType = StateTy; + + /// See AbstractAttribute::getState(...). + StateType &getState() override { return *this; } + + /// See AbstractAttribute::getState(...). + const AbstractState &getState() const override { return *this; } +}; + +/// Helper class that provides common functionality to manifest IR attributes. +template <Attribute::AttrKind AK, typename Base> +struct IRAttribute : public IRPosition, public Base { + IRAttribute(const IRPosition &IRP) : IRPosition(IRP) {} + ~IRAttribute() {} + + /// See AbstractAttribute::initialize(...). + virtual void initialize(Attributor &A) override { + if (hasAttr(getAttrKind())) { + this->getState().indicateOptimisticFixpoint(); + return; + } + + const IRPosition &IRP = this->getIRPosition(); + bool IsFnInterface = IRP.isFnInterfaceKind(); + const Function *FnScope = IRP.getAnchorScope(); + // TODO: Not all attributes require an exact definition. Find a way to + // enable deduction for some but not all attributes in case the + // definition might be changed at runtime, see also + // http://lists.llvm.org/pipermail/llvm-dev/2018-February/121275.html. + // TODO: We could always determine abstract attributes and if sufficient + // information was found we could duplicate the functions that do not + // have an exact definition. + if (IsFnInterface && (!FnScope || !FnScope->hasExactDefinition())) + this->getState().indicatePessimisticFixpoint(); + } + + /// See AbstractAttribute::manifest(...). + ChangeStatus manifest(Attributor &A) override { + SmallVector<Attribute, 4> DeducedAttrs; + getDeducedAttributes(getAnchorValue().getContext(), DeducedAttrs); + return IRAttributeManifest::manifestAttrs(A, getIRPosition(), DeducedAttrs); + } + + /// Return the kind that identifies the abstract attribute implementation. + Attribute::AttrKind getAttrKind() const { return AK; } + + /// Return the deduced attributes in \p Attrs. + virtual void getDeducedAttributes(LLVMContext &Ctx, + SmallVectorImpl<Attribute> &Attrs) const { + Attrs.emplace_back(Attribute::get(Ctx, getAttrKind())); + } + + /// Return an IR position, see struct IRPosition. + /// + ///{ + IRPosition &getIRPosition() override { return *this; } + const IRPosition &getIRPosition() const override { return *this; } + ///} +}; + /// Base struct for all "concrete attribute" deductions. /// /// The abstract attribute is a minimal interface that allows the Attributor to @@ -512,29 +1325,7 @@ struct BooleanState : public IntegerState { /// NOTE: The mechanics of adding a new "concrete" abstract attribute are /// described in the file comment. struct AbstractAttribute { - - /// The positions attributes can be manifested in. - enum ManifestPosition { - MP_ARGUMENT, ///< An attribute for a function argument. - MP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument. - MP_FUNCTION, ///< An attribute for a function as a whole. - MP_RETURNED, ///< An attribute for the function return value. - }; - - /// An abstract attribute associated with \p AssociatedVal and anchored at - /// \p AnchoredVal. - /// - /// \param AssociatedVal The value this abstract attribute is associated with. - /// \param AnchoredVal The value this abstract attributes is anchored at. - /// \param InfoCache Cached information accessible to the abstract attribute. - AbstractAttribute(Value *AssociatedVal, Value &AnchoredVal, - InformationCache &InfoCache) - : AssociatedVal(AssociatedVal), AnchoredVal(AnchoredVal), - InfoCache(InfoCache) {} - - /// An abstract attribute associated with and anchored at \p V. - AbstractAttribute(Value &V, InformationCache &InfoCache) - : AbstractAttribute(&V, V, InfoCache) {} + using StateType = AbstractState; /// Virtual destructor. virtual ~AbstractAttribute() {} @@ -550,47 +1341,11 @@ struct AbstractAttribute { virtual void initialize(Attributor &A) {} /// Return the internal abstract state for inspection. - virtual const AbstractState &getState() const = 0; - - /// Return the value this abstract attribute is anchored with. - /// - /// The anchored value might not be the associated value if the latter is not - /// sufficient to determine where arguments will be manifested. This is mostly - /// the case for call site arguments as the value is not sufficient to - /// pinpoint them. Instead, we can use the call site as an anchor. - /// - ///{ - Value &getAnchoredValue() { return AnchoredVal; } - const Value &getAnchoredValue() const { return AnchoredVal; } - ///} - - /// Return the llvm::Function surrounding the anchored value. - /// - ///{ - Function &getAnchorScope(); - const Function &getAnchorScope() const; - ///} - - /// Return the value this abstract attribute is associated with. - /// - /// The abstract state usually represents this value. - /// - ///{ - virtual Value *getAssociatedValue() { return AssociatedVal; } - virtual const Value *getAssociatedValue() const { return AssociatedVal; } - ///} - - /// Return the position this abstract state is manifested in. - virtual ManifestPosition getManifestPosition() const = 0; - - /// Return the kind that identifies the abstract attribute implementation. - virtual Attribute::AttrKind getAttrKind() const = 0; + virtual StateType &getState() = 0; + virtual const StateType &getState() const = 0; - /// Return the deduced attributes in \p Attrs. - virtual void getDeducedAttributes(SmallVectorImpl<Attribute> &Attrs) const { - LLVMContext &Ctx = AnchoredVal.getContext(); - Attrs.emplace_back(Attribute::get(Ctx, getAttrKind())); - } + /// Return an IR position, see struct IRPosition. + virtual const IRPosition &getIRPosition() const = 0; /// Helper functions, for debug purposes only. ///{ @@ -617,10 +1372,19 @@ protected: /// represented by the abstract attribute in the LLVM-IR. /// /// \Return CHANGED if the IR was altered, otherwise UNCHANGED. - virtual ChangeStatus manifest(Attributor &A); + virtual ChangeStatus manifest(Attributor &A) { + return ChangeStatus::UNCHANGED; + } - /// Return the internal abstract state for careful modification. - virtual AbstractState &getState() = 0; + /// Hook to enable custom statistic tracking, called after manifest that + /// resulted in a change if statistics are enabled. + /// + /// We require subclasses to provide an implementation so we remember to + /// add statistics for them. + virtual void trackStatistics() const = 0; + + /// Return an IR position, see struct IRPosition. + virtual IRPosition &getIRPosition() = 0; /// The actual update/transfer function which has to be implemented by the /// derived classes. @@ -630,15 +1394,6 @@ protected: /// /// \Return CHANGED if the internal state changed, otherwise UNCHANGED. virtual ChangeStatus updateImpl(Attributor &A) = 0; - - /// The value this abstract attribute is associated with. - Value *AssociatedVal; - - /// The value this abstract attribute is anchored at. - Value &AnchoredVal; - - /// The information cache accessible to this abstract attribute. - InformationCache &InfoCache; }; /// Forward declarations of output streams for debug purposes. @@ -646,8 +1401,10 @@ protected: ///{ raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA); raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S); -raw_ostream &operator<<(raw_ostream &OS, AbstractAttribute::ManifestPosition); +raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind); +raw_ostream &operator<<(raw_ostream &OS, const IRPosition &); raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State); +raw_ostream &operator<<(raw_ostream &OS, const IntegerState &S); ///} struct AttributorPass : public PassInfoMixin<AttributorPass> { @@ -661,129 +1418,531 @@ Pass *createAttributorLegacyPass(); /// ---------------------------------------------------------------------------- /// An abstract attribute for the returned values of a function. -struct AAReturnedValues : public AbstractAttribute { - /// See AbstractAttribute::AbstractAttribute(...). - AAReturnedValues(Function &F, InformationCache &InfoCache) - : AbstractAttribute(F, InfoCache) {} +struct AAReturnedValues + : public IRAttribute<Attribute::Returned, AbstractAttribute> { + AAReturnedValues(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return an assumed unique return value if a single candidate is found. If + /// there cannot be one, return a nullptr. If it is not clear yet, return the + /// Optional::NoneType. + Optional<Value *> getAssumedUniqueReturnValue(Attributor &A) const; /// Check \p Pred on all returned values. /// /// This method will evaluate \p Pred on returned values and return /// true if (1) all returned values are known, and (2) \p Pred returned true /// for all returned values. - virtual bool - checkForallReturnedValues(std::function<bool(Value &)> &Pred) const = 0; - - /// See AbstractAttribute::getAttrKind() - Attribute::AttrKind getAttrKind() const override { return ID; } - - /// The identifier used by the Attributor for this class of attributes. - static constexpr Attribute::AttrKind ID = Attribute::Returned; + /// + /// Note: Unlike the Attributor::checkForAllReturnedValuesAndReturnInsts + /// method, this one will not filter dead return instructions. + virtual bool checkForAllReturnedValuesAndReturnInsts( + const function_ref<bool(Value &, const SmallSetVector<ReturnInst *, 4> &)> + &Pred) const = 0; + + using iterator = + MapVector<Value *, SmallSetVector<ReturnInst *, 4>>::iterator; + using const_iterator = + MapVector<Value *, SmallSetVector<ReturnInst *, 4>>::const_iterator; + virtual llvm::iterator_range<iterator> returned_values() = 0; + virtual llvm::iterator_range<const_iterator> returned_values() const = 0; + + virtual size_t getNumReturnValues() const = 0; + virtual const SmallSetVector<CallBase *, 4> &getUnresolvedCalls() const = 0; + + /// Create an abstract attribute view for the position \p IRP. + static AAReturnedValues &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; }; -struct AANoUnwind : public AbstractAttribute { - /// An abstract interface for all nosync attributes. - AANoUnwind(Value &V, InformationCache &InfoCache) - : AbstractAttribute(V, InfoCache) {} - - /// See AbstractAttribute::getAttrKind()/ - Attribute::AttrKind getAttrKind() const override { return ID; } - - static constexpr Attribute::AttrKind ID = Attribute::NoUnwind; +struct AANoUnwind + : public IRAttribute<Attribute::NoUnwind, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoUnwind(const IRPosition &IRP) : IRAttribute(IRP) {} /// Returns true if nounwind is assumed. - virtual bool isAssumedNoUnwind() const = 0; + bool isAssumedNoUnwind() const { return getAssumed(); } /// Returns true if nounwind is known. - virtual bool isKnownNoUnwind() const = 0; -}; + bool isKnownNoUnwind() const { return getKnown(); } -struct AANoSync : public AbstractAttribute { - /// An abstract interface for all nosync attributes. - AANoSync(Value &V, InformationCache &InfoCache) - : AbstractAttribute(V, InfoCache) {} + /// Create an abstract attribute view for the position \p IRP. + static AANoUnwind &createForPosition(const IRPosition &IRP, Attributor &A); - /// See AbstractAttribute::getAttrKind(). - Attribute::AttrKind getAttrKind() const override { return ID; } + /// Unique ID (due to the unique address) + static const char ID; +}; - static constexpr Attribute::AttrKind ID = - Attribute::AttrKind(Attribute::NoSync); +struct AANoSync + : public IRAttribute<Attribute::NoSync, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoSync(const IRPosition &IRP) : IRAttribute(IRP) {} /// Returns true if "nosync" is assumed. - virtual bool isAssumedNoSync() const = 0; + bool isAssumedNoSync() const { return getAssumed(); } /// Returns true if "nosync" is known. - virtual bool isKnownNoSync() const = 0; -}; + bool isKnownNoSync() const { return getKnown(); } -/// An abstract interface for all nonnull attributes. -struct AANonNull : public AbstractAttribute { + /// Create an abstract attribute view for the position \p IRP. + static AANoSync &createForPosition(const IRPosition &IRP, Attributor &A); - /// See AbstractAttribute::AbstractAttribute(...). - AANonNull(Value &V, InformationCache &InfoCache) - : AbstractAttribute(V, InfoCache) {} + /// Unique ID (due to the unique address) + static const char ID; +}; - /// See AbstractAttribute::AbstractAttribute(...). - AANonNull(Value *AssociatedVal, Value &AnchoredValue, - InformationCache &InfoCache) - : AbstractAttribute(AssociatedVal, AnchoredValue, InfoCache) {} +/// An abstract interface for all nonnull attributes. +struct AANonNull + : public IRAttribute<Attribute::NonNull, + StateWrapper<BooleanState, AbstractAttribute>> { + AANonNull(const IRPosition &IRP) : IRAttribute(IRP) {} /// Return true if we assume that the underlying value is nonnull. - virtual bool isAssumedNonNull() const = 0; + bool isAssumedNonNull() const { return getAssumed(); } /// Return true if we know that underlying value is nonnull. - virtual bool isKnownNonNull() const = 0; + bool isKnownNonNull() const { return getKnown(); } - /// See AbastractState::getAttrKind(). - Attribute::AttrKind getAttrKind() const override { return ID; } + /// Create an abstract attribute view for the position \p IRP. + static AANonNull &createForPosition(const IRPosition &IRP, Attributor &A); - /// The identifier used by the Attributor for this class of attributes. - static constexpr Attribute::AttrKind ID = Attribute::NonNull; + /// Unique ID (due to the unique address) + static const char ID; }; /// An abstract attribute for norecurse. -struct AANoRecurse : public AbstractAttribute { +struct AANoRecurse + : public IRAttribute<Attribute::NoRecurse, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoRecurse(const IRPosition &IRP) : IRAttribute(IRP) {} - /// See AbstractAttribute::AbstractAttribute(...). - AANoRecurse(Value &V, InformationCache &InfoCache) - : AbstractAttribute(V, InfoCache) {} - - /// See AbstractAttribute::getAttrKind() - virtual Attribute::AttrKind getAttrKind() const override { - return Attribute::NoRecurse; - } + /// Return true if "norecurse" is assumed. + bool isAssumedNoRecurse() const { return getAssumed(); } /// Return true if "norecurse" is known. - virtual bool isKnownNoRecurse() const = 0; + bool isKnownNoRecurse() const { return getKnown(); } - /// Return true if "norecurse" is assumed. - virtual bool isAssumedNoRecurse() const = 0; + /// Create an abstract attribute view for the position \p IRP. + static AANoRecurse &createForPosition(const IRPosition &IRP, Attributor &A); - /// The identifier used by the Attributor for this class of attributes. - static constexpr Attribute::AttrKind ID = Attribute::NoRecurse; + /// Unique ID (due to the unique address) + static const char ID; }; /// An abstract attribute for willreturn. -struct AAWillReturn : public AbstractAttribute { +struct AAWillReturn + : public IRAttribute<Attribute::WillReturn, + StateWrapper<BooleanState, AbstractAttribute>> { + AAWillReturn(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return true if "willreturn" is assumed. + bool isAssumedWillReturn() const { return getAssumed(); } - /// See AbstractAttribute::AbstractAttribute(...). - AAWillReturn(Value &V, InformationCache &InfoCache) - : AbstractAttribute(V, InfoCache) {} + /// Return true if "willreturn" is known. + bool isKnownWillReturn() const { return getKnown(); } + + /// Create an abstract attribute view for the position \p IRP. + static AAWillReturn &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; - /// See AbstractAttribute::getAttrKind() - virtual Attribute::AttrKind getAttrKind() const override { - return Attribute::WillReturn; +/// An abstract interface for all noalias attributes. +struct AANoAlias + : public IRAttribute<Attribute::NoAlias, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoAlias(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return true if we assume that the underlying value is alias. + bool isAssumedNoAlias() const { return getAssumed(); } + + /// Return true if we know that underlying value is noalias. + bool isKnownNoAlias() const { return getKnown(); } + + /// Create an abstract attribute view for the position \p IRP. + static AANoAlias &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An AbstractAttribute for nofree. +struct AANoFree + : public IRAttribute<Attribute::NoFree, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoFree(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return true if "nofree" is assumed. + bool isAssumedNoFree() const { return getAssumed(); } + + /// Return true if "nofree" is known. + bool isKnownNoFree() const { return getKnown(); } + + /// Create an abstract attribute view for the position \p IRP. + static AANoFree &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An AbstractAttribute for noreturn. +struct AANoReturn + : public IRAttribute<Attribute::NoReturn, + StateWrapper<BooleanState, AbstractAttribute>> { + AANoReturn(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return true if the underlying object is assumed to never return. + bool isAssumedNoReturn() const { return getAssumed(); } + + /// Return true if the underlying object is known to never return. + bool isKnownNoReturn() const { return getKnown(); } + + /// Create an abstract attribute view for the position \p IRP. + static AANoReturn &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An abstract interface for liveness abstract attribute. +struct AAIsDead : public StateWrapper<BooleanState, AbstractAttribute>, + public IRPosition { + AAIsDead(const IRPosition &IRP) : IRPosition(IRP) {} + + /// Returns true if \p BB is assumed dead. + virtual bool isAssumedDead(const BasicBlock *BB) const = 0; + + /// Returns true if \p BB is known dead. + virtual bool isKnownDead(const BasicBlock *BB) const = 0; + + /// Returns true if \p I is assumed dead. + virtual bool isAssumedDead(const Instruction *I) const = 0; + + /// Returns true if \p I is known dead. + virtual bool isKnownDead(const Instruction *I) const = 0; + + /// This method is used to check if at least one instruction in a collection + /// of instructions is live. + template <typename T> bool isLiveInstSet(T begin, T end) const { + for (const auto &I : llvm::make_range(begin, end)) { + assert(I->getFunction() == getIRPosition().getAssociatedFunction() && + "Instruction must be in the same anchor scope function."); + + if (!isAssumedDead(I)) + return true; + } + + return false; } - /// Return true if "willreturn" is known. - virtual bool isKnownWillReturn() const = 0; + /// Return an IR position, see struct IRPosition. + /// + ///{ + IRPosition &getIRPosition() override { return *this; } + const IRPosition &getIRPosition() const override { return *this; } + ///} - /// Return true if "willreturn" is assumed. - virtual bool isAssumedWillReturn() const = 0; + /// Create an abstract attribute view for the position \p IRP. + static AAIsDead &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// State for dereferenceable attribute +struct DerefState : AbstractState { + + /// State representing for dereferenceable bytes. + IntegerState DerefBytesState; + + /// State representing that whether the value is globaly dereferenceable. + BooleanState GlobalState; + + /// See AbstractState::isValidState() + bool isValidState() const override { return DerefBytesState.isValidState(); } + + /// See AbstractState::isAtFixpoint() + bool isAtFixpoint() const override { + return !isValidState() || + (DerefBytesState.isAtFixpoint() && GlobalState.isAtFixpoint()); + } + + /// See AbstractState::indicateOptimisticFixpoint(...) + ChangeStatus indicateOptimisticFixpoint() override { + DerefBytesState.indicateOptimisticFixpoint(); + GlobalState.indicateOptimisticFixpoint(); + return ChangeStatus::UNCHANGED; + } + + /// See AbstractState::indicatePessimisticFixpoint(...) + ChangeStatus indicatePessimisticFixpoint() override { + DerefBytesState.indicatePessimisticFixpoint(); + GlobalState.indicatePessimisticFixpoint(); + return ChangeStatus::CHANGED; + } + + /// Update known dereferenceable bytes. + void takeKnownDerefBytesMaximum(uint64_t Bytes) { + DerefBytesState.takeKnownMaximum(Bytes); + } + + /// Update assumed dereferenceable bytes. + void takeAssumedDerefBytesMinimum(uint64_t Bytes) { + DerefBytesState.takeAssumedMinimum(Bytes); + } + + /// Equality for DerefState. + bool operator==(const DerefState &R) { + return this->DerefBytesState == R.DerefBytesState && + this->GlobalState == R.GlobalState; + } + + /// Inequality for IntegerState. + bool operator!=(const DerefState &R) { return !(*this == R); } + + /// See IntegerState::operator^= + DerefState operator^=(const DerefState &R) { + DerefBytesState ^= R.DerefBytesState; + GlobalState ^= R.GlobalState; + return *this; + } + + /// See IntegerState::operator+= + DerefState operator+=(const DerefState &R) { + DerefBytesState += R.DerefBytesState; + GlobalState += R.GlobalState; + return *this; + } + + /// See IntegerState::operator&= + DerefState operator&=(const DerefState &R) { + DerefBytesState &= R.DerefBytesState; + GlobalState &= R.GlobalState; + return *this; + } + + /// See IntegerState::operator|= + DerefState operator|=(const DerefState &R) { + DerefBytesState |= R.DerefBytesState; + GlobalState |= R.GlobalState; + return *this; + } - /// The identifier used by the Attributor for this class of attributes. - static constexpr Attribute::AttrKind ID = Attribute::WillReturn; +protected: + const AANonNull *NonNullAA = nullptr; +}; + +/// An abstract interface for all dereferenceable attribute. +struct AADereferenceable + : public IRAttribute<Attribute::Dereferenceable, + StateWrapper<DerefState, AbstractAttribute>> { + AADereferenceable(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return true if we assume that the underlying value is nonnull. + bool isAssumedNonNull() const { + return NonNullAA && NonNullAA->isAssumedNonNull(); + } + + /// Return true if we know that the underlying value is nonnull. + bool isKnownNonNull() const { + return NonNullAA && NonNullAA->isKnownNonNull(); + } + + /// Return true if we assume that underlying value is + /// dereferenceable(_or_null) globally. + bool isAssumedGlobal() const { return GlobalState.getAssumed(); } + + /// Return true if we know that underlying value is + /// dereferenceable(_or_null) globally. + bool isKnownGlobal() const { return GlobalState.getKnown(); } + + /// Return assumed dereferenceable bytes. + uint32_t getAssumedDereferenceableBytes() const { + return DerefBytesState.getAssumed(); + } + + /// Return known dereferenceable bytes. + uint32_t getKnownDereferenceableBytes() const { + return DerefBytesState.getKnown(); + } + + /// Create an abstract attribute view for the position \p IRP. + static AADereferenceable &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An abstract interface for all align attributes. +struct AAAlign + : public IRAttribute<Attribute::Alignment, + StateWrapper<IntegerState, AbstractAttribute>> { + AAAlign(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// Return assumed alignment. + unsigned getAssumedAlign() const { return getAssumed(); } + + /// Return known alignemnt. + unsigned getKnownAlign() const { return getKnown(); } + + /// Create an abstract attribute view for the position \p IRP. + static AAAlign &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An abstract interface for all nocapture attributes. +struct AANoCapture + : public IRAttribute<Attribute::NoCapture, + StateWrapper<IntegerState, AbstractAttribute>> { + AANoCapture(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// State encoding bits. A set bit in the state means the property holds. + /// NO_CAPTURE is the best possible state, 0 the worst possible state. + enum { + NOT_CAPTURED_IN_MEM = 1 << 0, + NOT_CAPTURED_IN_INT = 1 << 1, + NOT_CAPTURED_IN_RET = 1 << 2, + + /// If we do not capture the value in memory or through integers we can only + /// communicate it back as a derived pointer. + NO_CAPTURE_MAYBE_RETURNED = NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT, + + /// If we do not capture the value in memory, through integers, or as a + /// derived pointer we know it is not captured. + NO_CAPTURE = + NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT | NOT_CAPTURED_IN_RET, + }; + + /// Return true if we know that the underlying value is not captured in its + /// respective scope. + bool isKnownNoCapture() const { return isKnown(NO_CAPTURE); } + + /// Return true if we assume that the underlying value is not captured in its + /// respective scope. + bool isAssumedNoCapture() const { return isAssumed(NO_CAPTURE); } + + /// Return true if we know that the underlying value is not captured in its + /// respective scope but we allow it to escape through a "return". + bool isKnownNoCaptureMaybeReturned() const { + return isKnown(NO_CAPTURE_MAYBE_RETURNED); + } + + /// Return true if we assume that the underlying value is not captured in its + /// respective scope but we allow it to escape through a "return". + bool isAssumedNoCaptureMaybeReturned() const { + return isAssumed(NO_CAPTURE_MAYBE_RETURNED); + } + + /// Create an abstract attribute view for the position \p IRP. + static AANoCapture &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; }; + +/// An abstract interface for value simplify abstract attribute. +struct AAValueSimplify : public StateWrapper<BooleanState, AbstractAttribute>, + public IRPosition { + AAValueSimplify(const IRPosition &IRP) : IRPosition(IRP) {} + + /// Return an IR position, see struct IRPosition. + /// + ///{ + IRPosition &getIRPosition() { return *this; } + const IRPosition &getIRPosition() const { return *this; } + ///} + + /// Return an assumed simplified value if a single candidate is found. If + /// there cannot be one, return original value. If it is not clear yet, return + /// the Optional::NoneType. + virtual Optional<Value *> getAssumedSimplifiedValue(Attributor &A) const = 0; + + /// Create an abstract attribute view for the position \p IRP. + static AAValueSimplify &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute>, + public IRPosition { + AAHeapToStack(const IRPosition &IRP) : IRPosition(IRP) {} + + /// Returns true if HeapToStack conversion is assumed to be possible. + bool isAssumedHeapToStack() const { return getAssumed(); } + + /// Returns true if HeapToStack conversion is known to be possible. + bool isKnownHeapToStack() const { return getKnown(); } + + /// Return an IR position, see struct IRPosition. + /// + ///{ + IRPosition &getIRPosition() { return *this; } + const IRPosition &getIRPosition() const { return *this; } + ///} + + /// Create an abstract attribute view for the position \p IRP. + static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An abstract interface for all memory related attributes. +struct AAMemoryBehavior + : public IRAttribute<Attribute::ReadNone, + StateWrapper<IntegerState, AbstractAttribute>> { + AAMemoryBehavior(const IRPosition &IRP) : IRAttribute(IRP) {} + + /// State encoding bits. A set bit in the state means the property holds. + /// BEST_STATE is the best possible state, 0 the worst possible state. + enum { + NO_READS = 1 << 0, + NO_WRITES = 1 << 1, + NO_ACCESSES = NO_READS | NO_WRITES, + + BEST_STATE = NO_ACCESSES, + }; + + /// Return true if we know that the underlying value is not read or accessed + /// in its respective scope. + bool isKnownReadNone() const { return isKnown(NO_ACCESSES); } + + /// Return true if we assume that the underlying value is not read or accessed + /// in its respective scope. + bool isAssumedReadNone() const { return isAssumed(NO_ACCESSES); } + + /// Return true if we know that the underlying value is not accessed + /// (=written) in its respective scope. + bool isKnownReadOnly() const { return isKnown(NO_WRITES); } + + /// Return true if we assume that the underlying value is not accessed + /// (=written) in its respective scope. + bool isAssumedReadOnly() const { return isAssumed(NO_WRITES); } + + /// Return true if we know that the underlying value is not read in its + /// respective scope. + bool isKnownWriteOnly() const { return isKnown(NO_READS); } + + /// Return true if we assume that the underlying value is not read in its + /// respective scope. + bool isAssumedWriteOnly() const { return isAssumed(NO_READS); } + + /// Create an abstract attribute view for the position \p IRP. + static AAMemoryBehavior &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H diff --git a/include/llvm/Transforms/IPO/GlobalDCE.h b/include/llvm/Transforms/IPO/GlobalDCE.h index c434484d1ae3..0a6851849e7e 100644 --- a/include/llvm/Transforms/IPO/GlobalDCE.h +++ b/include/llvm/Transforms/IPO/GlobalDCE.h @@ -43,11 +43,25 @@ private: /// Comdat -> Globals in that Comdat section. std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers; + /// !type metadata -> set of (vtable, offset) pairs + DenseMap<Metadata *, SmallSet<std::pair<GlobalVariable *, uint64_t>, 4>> + TypeIdMap; + + // Global variables which are vtables, and which we have enough information + // about to safely do dead virtual function elimination. + SmallPtrSet<GlobalValue *, 32> VFESafeVTables; + void UpdateGVDependencies(GlobalValue &GV); void MarkLive(GlobalValue &GV, SmallVectorImpl<GlobalValue *> *Updates = nullptr); bool RemoveUnusedGlobalValue(GlobalValue &GV); + // Dead virtual function elimination. + void AddVirtualFunctionDependencies(Module &M); + void ScanVTables(Module &M); + void ScanTypeCheckedLoadIntrinsics(Module &M); + void ScanVTableLoad(Function *Caller, Metadata *TypeId, uint64_t CallOffset); + void ComputeDependencies(Value *V, SmallPtrSetImpl<GlobalValue *> &U); }; diff --git a/include/llvm/Transforms/IPO/HotColdSplitting.h b/include/llvm/Transforms/IPO/HotColdSplitting.h index 73668844590d..8c3049fbaac4 100644 --- a/include/llvm/Transforms/IPO/HotColdSplitting.h +++ b/include/llvm/Transforms/IPO/HotColdSplitting.h @@ -17,6 +17,45 @@ namespace llvm { class Module; +class ProfileSummaryInfo; +class BlockFrequencyInfo; +class TargetTransformInfo; +class OptimizationRemarkEmitter; +class AssumptionCache; +class DominatorTree; +class CodeExtractorAnalysisCache; + +/// A sequence of basic blocks. +/// +/// A 0-sized SmallVector is slightly cheaper to move than a std::vector. +using BlockSequence = SmallVector<BasicBlock *, 0>; + +class HotColdSplitting { +public: + HotColdSplitting(ProfileSummaryInfo *ProfSI, + function_ref<BlockFrequencyInfo *(Function &)> GBFI, + function_ref<TargetTransformInfo &(Function &)> GTTI, + std::function<OptimizationRemarkEmitter &(Function &)> *GORE, + function_ref<AssumptionCache *(Function &)> LAC) + : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE), LookupAC(LAC) {} + bool run(Module &M); + +private: + bool isFunctionCold(const Function &F) const; + bool shouldOutlineFrom(const Function &F) const; + bool outlineColdRegions(Function &F, bool HasProfileSummary); + Function *extractColdRegion(const BlockSequence &Region, + const CodeExtractorAnalysisCache &CEAC, + DominatorTree &DT, BlockFrequencyInfo *BFI, + TargetTransformInfo &TTI, + OptimizationRemarkEmitter &ORE, + AssumptionCache *AC, unsigned Count); + ProfileSummaryInfo *PSI; + function_ref<BlockFrequencyInfo *(Function &)> GetBFI; + function_ref<TargetTransformInfo &(Function &)> GetTTI; + std::function<OptimizationRemarkEmitter &(Function &)> *GetORE; + function_ref<AssumptionCache *(Function &)> LookupAC; +}; /// Pass to outline cold regions. class HotColdSplittingPass : public PassInfoMixin<HotColdSplittingPass> { diff --git a/include/llvm/Transforms/IPO/LowerTypeTests.h b/include/llvm/Transforms/IPO/LowerTypeTests.h index 39b23f5957db..3c2bb65b9552 100644 --- a/include/llvm/Transforms/IPO/LowerTypeTests.h +++ b/include/llvm/Transforms/IPO/LowerTypeTests.h @@ -193,6 +193,8 @@ struct ByteArrayBuilder { uint64_t &AllocByteOffset, uint8_t &AllocMask); }; +bool isJumpTableCanonical(Function *F); + } // end namespace lowertypetests class LowerTypeTestsPass : public PassInfoMixin<LowerTypeTestsPass> { diff --git a/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/include/llvm/Transforms/IPO/WholeProgramDevirt.h index 509fcc867060..22435e4ed1e5 100644 --- a/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -16,8 +16,10 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/IPO/FunctionImport.h" #include <cassert> #include <cstdint> +#include <set> #include <utility> #include <vector> @@ -28,6 +30,7 @@ template <typename T> class MutableArrayRef; class Function; class GlobalVariable; class ModuleSummaryIndex; +struct ValueInfo; namespace wholeprogramdevirt { @@ -228,6 +231,29 @@ struct WholeProgramDevirtPass : public PassInfoMixin<WholeProgramDevirtPass> { PreservedAnalyses run(Module &M, ModuleAnalysisManager &); }; +struct VTableSlotSummary { + StringRef TypeID; + uint64_t ByteOffset; +}; + +/// Perform index-based whole program devirtualization on the \p Summary +/// index. Any devirtualized targets used by a type test in another module +/// are added to the \p ExportedGUIDs set. For any local devirtualized targets +/// only used within the defining module, the information necessary for +/// locating the corresponding WPD resolution is recorded for the ValueInfo +/// in case it is exported by cross module importing (in which case the +/// devirtualized target name will need adjustment). +void runWholeProgramDevirtOnIndex( + ModuleSummaryIndex &Summary, std::set<GlobalValue::GUID> &ExportedGUIDs, + std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap); + +/// Call after cross-module importing to update the recorded single impl +/// devirt target names for any locals that were exported. +void updateIndexWPDForExports( + ModuleSummaryIndex &Summary, + function_ref<bool(StringRef, GlobalValue::GUID)> isExported, + std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap); + } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_WHOLEPROGRAMDEVIRT_H diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h index 8b70d2926ae9..fcad1e11895f 100644 --- a/include/llvm/Transforms/Instrumentation.h +++ b/include/llvm/Transforms/Instrumentation.h @@ -181,10 +181,6 @@ struct SanitizerCoverageOptions { SanitizerCoverageOptions() = default; }; -// Insert SanitizerCoverage instrumentation. -ModulePass *createSanitizerCoverageModulePass( - const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); - /// Calculate what to divide by to scale counts. /// /// Given the maximum count, calculate a divisor that will scale all the diff --git a/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/include/llvm/Transforms/Instrumentation/InstrProfiling.h index 8f76d4a1ce55..2e0fae527b15 100644 --- a/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -39,13 +39,14 @@ public: : Options(Options), IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - bool run(Module &M, const TargetLibraryInfo &TLI); + bool run(Module &M, + std::function<const TargetLibraryInfo &(Function &F)> GetTLI); private: InstrProfOptions Options; Module *M; Triple TT; - const TargetLibraryInfo *TLI; + std::function<const TargetLibraryInfo &(Function &F)> GetTLI; struct PerFunctionProfileData { uint32_t NumValueSites[IPVK_Last + 1]; GlobalVariable *RegionCounters = nullptr; diff --git a/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h index 0739d9e58a61..01a86ee3f1fd 100644 --- a/include/llvm/Transforms/Instrumentation/MemorySanitizer.h +++ b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h @@ -19,12 +19,11 @@ namespace llvm { struct MemorySanitizerOptions { - MemorySanitizerOptions() = default; - MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel) - : TrackOrigins(TrackOrigins), Recover(Recover), Kernel(Kernel) {} - int TrackOrigins = 0; - bool Recover = false; - bool Kernel = false; + MemorySanitizerOptions() : MemorySanitizerOptions(0, false, false){}; + MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel); + bool Kernel; + int TrackOrigins; + bool Recover; }; // Insert MemorySanitizer instrumentation (detection of uninitialized reads) @@ -41,6 +40,7 @@ struct MemorySanitizerPass : public PassInfoMixin<MemorySanitizerPass> { MemorySanitizerPass(MemorySanitizerOptions Options) : Options(Options) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: MemorySanitizerOptions Options; diff --git a/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h b/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h new file mode 100644 index 000000000000..85a43ff86f2e --- /dev/null +++ b/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h @@ -0,0 +1,47 @@ +//===--------- Definition of the SanitizerCoverage class --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the SanitizerCoverage class which is a port of the legacy +// SanitizerCoverage pass to use the new PassManager infrastructure. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERCOVERAGE_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERCOVERAGE_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Instrumentation.h" + +namespace llvm { + +/// This is the ModuleSanitizerCoverage pass used in the new pass manager. The +/// pass instruments functions for coverage, adds initialization calls to the +/// module for trace PC guards and 8bit counters if they are requested, and +/// appends globals to llvm.compiler.used. +class ModuleSanitizerCoveragePass + : public PassInfoMixin<ModuleSanitizerCoveragePass> { +public: + explicit ModuleSanitizerCoveragePass( + SanitizerCoverageOptions Options = SanitizerCoverageOptions()) + : Options(Options) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + SanitizerCoverageOptions Options; +}; + +// Insert SanitizerCoverage instrumentation. +ModulePass *createModuleSanitizerCoverageLegacyPassPass( + const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); + +} // namespace llvm + +#endif diff --git a/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h index b4e7d9924ff6..ce0e46745abb 100644 --- a/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h +++ b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h @@ -27,6 +27,8 @@ FunctionPass *createThreadSanitizerLegacyPassPass(); /// yet, the pass inserts the declarations. Otherwise the existing globals are struct ThreadSanitizerPass : public PassInfoMixin<ThreadSanitizerPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; + } // namespace llvm #endif /* LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H */ diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h index f9360b5ee2c8..f06230b6f366 100644 --- a/include/llvm/Transforms/Scalar.h +++ b/include/llvm/Transforms/Scalar.h @@ -308,7 +308,7 @@ FunctionPass *createGVNSinkPass(); // MergedLoadStoreMotion - This pass merges loads and stores in diamonds. Loads // are hoisted into the header, while stores sink into the footer. // -FunctionPass *createMergedLoadStoreMotionPass(); +FunctionPass *createMergedLoadStoreMotionPass(bool SplitFooterBB = false); //===----------------------------------------------------------------------===// // @@ -397,6 +397,13 @@ FunctionPass *createLowerExpectIntrinsicPass(); //===----------------------------------------------------------------------===// // +// LowerConstantIntrinsicss - Expand any remaining llvm.objectsize and +// llvm.is.constant intrinsic calls, even for the unknown cases. +// +FunctionPass *createLowerConstantIntrinsicsPass(); + +//===----------------------------------------------------------------------===// +// // PartiallyInlineLibCalls - Tries to inline the fast path of library // calls such as sqrt. // diff --git a/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/include/llvm/Transforms/Scalar/CallSiteSplitting.h index b6055639e8a8..74cbf84b64b2 100644 --- a/include/llvm/Transforms/Scalar/CallSiteSplitting.h +++ b/include/llvm/Transforms/Scalar/CallSiteSplitting.h @@ -9,13 +9,8 @@ #ifndef LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H #define LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H -#include "llvm/ADT/SetVector.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/Compiler.h" -#include <vector> namespace llvm { diff --git a/include/llvm/Transforms/Scalar/ConstantHoisting.h b/include/llvm/Transforms/Scalar/ConstantHoisting.h index 6b0fc9c1dd07..39039b093241 100644 --- a/include/llvm/Transforms/Scalar/ConstantHoisting.h +++ b/include/llvm/Transforms/Scalar/ConstantHoisting.h @@ -37,7 +37,9 @@ #define LLVM_TRANSFORMS_SCALAR_CONSTANTHOISTING_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" @@ -154,21 +156,21 @@ private: /// Keeps track of constant candidates found in the function. using ConstCandVecType = std::vector<consthoist::ConstantCandidate>; - using GVCandVecMapType = DenseMap<GlobalVariable *, ConstCandVecType>; + using GVCandVecMapType = MapVector<GlobalVariable *, ConstCandVecType>; ConstCandVecType ConstIntCandVec; GVCandVecMapType ConstGEPCandMap; /// These are the final constants we decided to hoist. using ConstInfoVecType = SmallVector<consthoist::ConstantInfo, 8>; - using GVInfoVecMapType = DenseMap<GlobalVariable *, ConstInfoVecType>; + using GVInfoVecMapType = MapVector<GlobalVariable *, ConstInfoVecType>; ConstInfoVecType ConstIntInfoVec; GVInfoVecMapType ConstGEPInfoMap; /// Keep track of cast instructions we already cloned. - SmallDenseMap<Instruction *, Instruction *> ClonedCastMap; + MapVector<Instruction *, Instruction *> ClonedCastMap; Instruction *findMatInsertPt(Instruction *Inst, unsigned Idx = ~0U) const; - SmallPtrSet<Instruction *, 8> + SetVector<Instruction *> findConstantInsertionPoint(const consthoist::ConstantInfo &ConstInfo) const; void collectConstantCandidates(ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx, diff --git a/include/llvm/Transforms/Scalar/Float2Int.h b/include/llvm/Transforms/Scalar/Float2Int.h index 06aeb8322527..f04b98a19d82 100644 --- a/include/llvm/Transforms/Scalar/Float2Int.h +++ b/include/llvm/Transforms/Scalar/Float2Int.h @@ -17,6 +17,7 @@ #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/MapVector.h" #include "llvm/IR/ConstantRange.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" @@ -26,10 +27,11 @@ public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); // Glue for old PM. - bool runImpl(Function &F); + bool runImpl(Function &F, const DominatorTree &DT); private: - void findRoots(Function &F, SmallPtrSet<Instruction *, 8> &Roots); + void findRoots(Function &F, const DominatorTree &DT, + SmallPtrSet<Instruction *, 8> &Roots); void seen(Instruction *I, ConstantRange R); ConstantRange badRange(); ConstantRange unknownRange(); diff --git a/include/llvm/Transforms/Scalar/GVN.h b/include/llvm/Transforms/Scalar/GVN.h index 9fe00a9e7f2d..8a64768af6b5 100644 --- a/include/llvm/Transforms/Scalar/GVN.h +++ b/include/llvm/Transforms/Scalar/GVN.h @@ -120,6 +120,8 @@ public: uint32_t lookupOrAddCall(CallInst *C); uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock, uint32_t Num, GVN &Gvn); + bool areCallValsEqual(uint32_t Num, uint32_t NewNum, const BasicBlock *Pred, + const BasicBlock *PhiBlock, GVN &Gvn); std::pair<uint32_t, bool> assignExpNewValueNum(Expression &exp); bool areAllValsInBB(uint32_t num, const BasicBlock *BB, GVN &Gvn); @@ -159,6 +161,7 @@ private: SetVector<BasicBlock *> DeadBlocks; OptimizationRemarkEmitter *ORE; ImplicitControlFlowTracking *ICF; + LoopInfo *LI; ValueTable VN; @@ -175,7 +178,7 @@ private: // Block-local map of equivalent values to their leader, does not // propagate to any successors. Entries added mid-block are applied // to the remaining instructions in the block. - SmallMapVector<Value *, Constant *, 4> ReplaceWithConstMap; + SmallMapVector<Value *, Value *, 4> ReplaceOperandsWithMap; SmallVector<Instruction *, 8> InstrsToErase; // Map the block to reversed postorder traversal number. It is used to @@ -280,7 +283,7 @@ private: void verifyRemoved(const Instruction *I) const; bool splitCriticalEdges(); BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ); - bool replaceOperandsWithConsts(Instruction *I) const; + bool replaceOperandsForInBlockEquality(Instruction *I) const; bool propagateEquality(Value *LHS, Value *RHS, const BasicBlockEdge &Root, bool DominatesByEdge); bool processFoldableCondBr(BranchInst *BI); diff --git a/include/llvm/Transforms/Scalar/GVNExpression.h b/include/llvm/Transforms/Scalar/GVNExpression.h index 3dc4515f85a1..1600d1af3242 100644 --- a/include/llvm/Transforms/Scalar/GVNExpression.h +++ b/include/llvm/Transforms/Scalar/GVNExpression.h @@ -323,7 +323,7 @@ public: class LoadExpression final : public MemoryExpression { private: LoadInst *Load; - unsigned Alignment; + MaybeAlign Alignment; public: LoadExpression(unsigned NumOperands, LoadInst *L, @@ -333,7 +333,8 @@ public: LoadExpression(enum ExpressionType EType, unsigned NumOperands, LoadInst *L, const MemoryAccess *MemoryLeader) : MemoryExpression(NumOperands, EType, MemoryLeader), Load(L) { - Alignment = L ? L->getAlignment() : 0; + if (L) + Alignment = MaybeAlign(L->getAlignment()); } LoadExpression() = delete; @@ -348,8 +349,8 @@ public: LoadInst *getLoadInst() const { return Load; } void setLoadInst(LoadInst *L) { Load = L; } - unsigned getAlignment() const { return Alignment; } - void setAlignment(unsigned Align) { Alignment = Align; } + MaybeAlign getAlignment() const { return Alignment; } + void setAlignment(MaybeAlign Align) { Alignment = Align; } bool equals(const Expression &Other) const override; bool exactlyEquals(const Expression &Other) const override { diff --git a/include/llvm/Transforms/Scalar/LoopPassManager.h b/include/llvm/Transforms/Scalar/LoopPassManager.h index 61ec58585fd0..aed764855b2e 100644 --- a/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -263,8 +263,10 @@ template <typename LoopPassT> class FunctionToLoopPassAdaptor : public PassInfoMixin<FunctionToLoopPassAdaptor<LoopPassT>> { public: - explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool DebugLogging = false) - : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging) { + explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, + bool DebugLogging = false) + : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging), + UseMemorySSA(UseMemorySSA) { LoopCanonicalizationFPM.addPass(LoopSimplifyPass()); LoopCanonicalizationFPM.addPass(LCSSAPass()); } @@ -293,7 +295,7 @@ public: return PA; // Get the analysis results needed by loop passes. - MemorySSA *MSSA = EnableMSSALoopDependency + MemorySSA *MSSA = UseMemorySSA ? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA()) : nullptr; LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F), @@ -310,8 +312,10 @@ public: // LoopStandardAnalysisResults object. The loop analyses cached in this // manager have access to those analysis results and so it must invalidate // itself when they go away. - LoopAnalysisManager &LAM = - AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager(); + auto &LAMFP = AM.getResult<LoopAnalysisManagerFunctionProxy>(F); + if (UseMemorySSA) + LAMFP.markMSSAUsed(); + LoopAnalysisManager &LAM = LAMFP.getManager(); // A postorder worklist of loops to process. SmallPriorityWorklist<Loop *, 4> Worklist; @@ -382,7 +386,7 @@ public: PA.preserve<DominatorTreeAnalysis>(); PA.preserve<LoopAnalysis>(); PA.preserve<ScalarEvolutionAnalysis>(); - if (EnableMSSALoopDependency) + if (UseMemorySSA) PA.preserve<MemorySSAAnalysis>(); // FIXME: What we really want to do here is preserve an AA category, but // that concept doesn't exist yet. @@ -397,14 +401,18 @@ private: LoopPassT Pass; FunctionPassManager LoopCanonicalizationFPM; + + bool UseMemorySSA = false; }; /// A function to deduce a loop pass type and wrap it in the templated /// adaptor. template <typename LoopPassT> FunctionToLoopPassAdaptor<LoopPassT> -createFunctionToLoopPassAdaptor(LoopPassT Pass, bool DebugLogging = false) { - return FunctionToLoopPassAdaptor<LoopPassT>(std::move(Pass), DebugLogging); +createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, + bool DebugLogging = false) { + return FunctionToLoopPassAdaptor<LoopPassT>(std::move(Pass), UseMemorySSA, + DebugLogging); } /// Pass for printing a loop's contents as textual IR. diff --git a/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/include/llvm/Transforms/Scalar/LoopUnrollPass.h index a84d889a83ad..afeb1f1da029 100644 --- a/include/llvm/Transforms/Scalar/LoopUnrollPass.h +++ b/include/llvm/Transforms/Scalar/LoopUnrollPass.h @@ -62,6 +62,8 @@ struct LoopUnrollOptions { Optional<bool> AllowPeeling; Optional<bool> AllowRuntime; Optional<bool> AllowUpperBound; + Optional<bool> AllowProfileBasedPeeling; + Optional<unsigned> FullUnrollMaxCount; int OptLevel; /// If false, use a cost model to determine whether unrolling of a loop is @@ -110,6 +112,18 @@ struct LoopUnrollOptions { OptLevel = O; return *this; } + + // Enables or disables loop peeling basing on profile. + LoopUnrollOptions &setProfileBasedPeeling(int O) { + AllowProfileBasedPeeling = O; + return *this; + } + + // Sets the max full unroll count. + LoopUnrollOptions &setFullUnrollMaxCount(unsigned O) { + FullUnrollMaxCount = O; + return *this; + } }; /// Loop unroll pass that will support both full and partial unrolling. diff --git a/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h b/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h new file mode 100644 index 000000000000..a5ad4a2192a0 --- /dev/null +++ b/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h @@ -0,0 +1,41 @@ +//===- LowerConstantIntrinsics.h - Lower constant int. pass -*- C++ -*-========// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// The header file for the LowerConstantIntrinsics pass as used by the new pass +/// manager. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERCONSTANTINTRINSICS_H +#define LLVM_TRANSFORMS_SCALAR_LOWERCONSTANTINTRINSICS_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +struct LowerConstantIntrinsicsPass : + PassInfoMixin<LowerConstantIntrinsicsPass> { +public: + explicit LowerConstantIntrinsicsPass() {} + + /// Run the pass over the function. + /// + /// This will lower all remaining 'objectsize' and 'is.constant'` + /// intrinsic calls in this function, even when the argument has no known + /// size or is not a constant respectively. The resulting constant is + /// propagated and conditional branches are resolved where possible. + /// This complements the Instruction Simplification and + /// Instruction Combination passes of the optimized pass chain. + PreservedAnalyses run(Function &F, FunctionAnalysisManager &); +}; + +} + +#endif diff --git a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h index 9071a56532f8..c5f6d6e0e8bd 100644 --- a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h +++ b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h @@ -27,12 +27,28 @@ #include "llvm/IR/PassManager.h" namespace llvm { +struct MergedLoadStoreMotionOptions { + bool SplitFooterBB; + MergedLoadStoreMotionOptions(bool SplitFooterBB = false) + : SplitFooterBB(SplitFooterBB) {} + + MergedLoadStoreMotionOptions &splitFooterBB(bool SFBB) { + SplitFooterBB = SFBB; + return *this; + } +}; + class MergedLoadStoreMotionPass : public PassInfoMixin<MergedLoadStoreMotionPass> { + MergedLoadStoreMotionOptions Options; + public: + MergedLoadStoreMotionPass() + : MergedLoadStoreMotionPass(MergedLoadStoreMotionOptions()) {} + MergedLoadStoreMotionPass(const MergedLoadStoreMotionOptions &PassOptions) + : Options(PassOptions) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; - } #endif // LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H diff --git a/include/llvm/Transforms/Scalar/Reassociate.h b/include/llvm/Transforms/Scalar/Reassociate.h index 2db8d8ce309c..d5b175eff0e6 100644 --- a/include/llvm/Transforms/Scalar/Reassociate.h +++ b/include/llvm/Transforms/Scalar/Reassociate.h @@ -122,7 +122,9 @@ private: void EraseInst(Instruction *I); void RecursivelyEraseDeadInsts(Instruction *I, OrderedSet &Insts); void OptimizeInst(Instruction *I); - Instruction *canonicalizeNegConstExpr(Instruction *I); + Instruction *canonicalizeNegFPConstantsForOp(Instruction *I, Instruction *Op, + Value *OtherOp); + Instruction *canonicalizeNegFPConstants(Instruction *I); void BuildPairMap(ReversePostOrderTraversal<Function *> &RPOT); }; diff --git a/include/llvm/Transforms/Scalar/SCCP.h b/include/llvm/Transforms/Scalar/SCCP.h index 0ffd983eb3e0..45e674a20a16 100644 --- a/include/llvm/Transforms/Scalar/SCCP.h +++ b/include/llvm/Transforms/Scalar/SCCP.h @@ -45,7 +45,8 @@ struct AnalysisResultsForFn { PostDominatorTree *PDT; }; -bool runIPSCCP(Module &M, const DataLayout &DL, const TargetLibraryInfo *TLI, +bool runIPSCCP(Module &M, const DataLayout &DL, + std::function<const TargetLibraryInfo &(Function &)> GetTLI, function_ref<AnalysisResultsForFn(Function &)> getAnalysis); } // end namespace llvm diff --git a/include/llvm/Transforms/Utils/BasicBlockUtils.h b/include/llvm/Transforms/Utils/BasicBlockUtils.h index 4d861ffe9a31..698e57fd0394 100644 --- a/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -83,10 +83,16 @@ bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI = nullptr); /// Attempts to merge a block into its predecessor, if possible. The return /// value indicates success or failure. +/// By default do not merge blocks if BB's predecessor has multiple successors. +/// If PredecessorWithTwoSuccessors = true, the blocks can only be merged +/// if BB's Pred has a branch to BB and to AnotherBB, and BB has a single +/// successor Sing. In this case the branch will be updated with Sing instead of +/// BB, and BB will still be merged into its predecessor and removed. bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr, MemorySSAUpdater *MSSAU = nullptr, - MemoryDependenceResults *MemDep = nullptr); + MemoryDependenceResults *MemDep = nullptr, + bool PredecessorWithTwoSuccessors = false); /// Replace all uses of an instruction (specified by BI) with a value, then /// remove and delete the original instruction. @@ -222,7 +228,8 @@ BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To, /// info is updated. BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT = nullptr, LoopInfo *LI = nullptr, - MemorySSAUpdater *MSSAU = nullptr); + MemorySSAUpdater *MSSAU = nullptr, + const Twine &BBName = ""); /// This method introduces at least one new basic block into the function and /// moves some of the predecessors of BB to be predecessors of the new block. diff --git a/include/llvm/Transforms/Utils/BuildLibCalls.h b/include/llvm/Transforms/Utils/BuildLibCalls.h index 8421c31a36da..3d15b2a7bf2a 100644 --- a/include/llvm/Transforms/Utils/BuildLibCalls.h +++ b/include/llvm/Transforms/Utils/BuildLibCalls.h @@ -30,17 +30,16 @@ namespace llvm { bool inferLibFuncAttributes(Function &F, const TargetLibraryInfo &TLI); bool inferLibFuncAttributes(Module *M, StringRef Name, const TargetLibraryInfo &TLI); - /// Check whether the overloaded unary floating point function + /// Check whether the overloaded floating point function /// corresponding to \a Ty is available. - bool hasUnaryFloatFn(const TargetLibraryInfo *TLI, Type *Ty, - LibFunc DoubleFn, LibFunc FloatFn, - LibFunc LongDoubleFn); + bool hasFloatFn(const TargetLibraryInfo *TLI, Type *Ty, + LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn); - /// Get the name of the overloaded unary floating point function + /// Get the name of the overloaded floating point function /// corresponding to \a Ty. - StringRef getUnaryFloatFn(const TargetLibraryInfo *TLI, Type *Ty, - LibFunc DoubleFn, LibFunc FloatFn, - LibFunc LongDoubleFn); + StringRef getFloatFnName(const TargetLibraryInfo *TLI, Type *Ty, + LibFunc DoubleFn, LibFunc FloatFn, + LibFunc LongDoubleFn); /// Return V if it is an i8*, otherwise cast it to i8*. Value *castToCStr(Value *V, IRBuilder<> &B); @@ -51,6 +50,11 @@ namespace llvm { Value *emitStrLen(Value *Ptr, IRBuilder<> &B, const DataLayout &DL, const TargetLibraryInfo *TLI); + /// Emit a call to the strdup function to the builder, for the specified + /// pointer. Ptr is required to be some pointer type, and the return value has + /// 'i8*' type. + Value *emitStrDup(Value *Ptr, IRBuilder<> &B, const TargetLibraryInfo *TLI); + /// Emit a call to the strnlen function to the builder, for the specified /// pointer. Ptr is required to be some pointer type, MaxLen must be of size_t /// type, and the return value has 'intptr_t' type. @@ -164,6 +168,13 @@ namespace llvm { Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2, StringRef Name, IRBuilder<> &B, const AttributeList &Attrs); + /// Emit a call to the binary function DoubleFn, FloatFn or LongDoubleFn, + /// depending of the type of Op1. + Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2, + const TargetLibraryInfo *TLI, LibFunc DoubleFn, + LibFunc FloatFn, LibFunc LongDoubleFn, + IRBuilder<> &B, const AttributeList &Attrs); + /// Emit a call to the putchar function. This assumes that Char is an integer. Value *emitPutChar(Value *Char, IRBuilder<> &B, const TargetLibraryInfo *TLI); diff --git a/include/llvm/Transforms/Utils/BypassSlowDivision.h b/include/llvm/Transforms/Utils/BypassSlowDivision.h index 471055921fa8..bd98c902d1ab 100644 --- a/include/llvm/Transforms/Utils/BypassSlowDivision.h +++ b/include/llvm/Transforms/Utils/BypassSlowDivision.h @@ -19,6 +19,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" +#include "llvm/IR/ValueHandle.h" #include <cstdint> namespace llvm { @@ -28,8 +29,10 @@ class Value; struct DivRemMapKey { bool SignedOp; - Value *Dividend; - Value *Divisor; + AssertingVH<Value> Dividend; + AssertingVH<Value> Divisor; + + DivRemMapKey() = default; DivRemMapKey(bool InSignedOp, Value *InDividend, Value *InDivisor) : SignedOp(InSignedOp), Dividend(InDividend), Divisor(InDivisor) {} @@ -50,8 +53,10 @@ template <> struct DenseMapInfo<DivRemMapKey> { } static unsigned getHashValue(const DivRemMapKey &Val) { - return (unsigned)(reinterpret_cast<uintptr_t>(Val.Dividend) ^ - reinterpret_cast<uintptr_t>(Val.Divisor)) ^ + return (unsigned)(reinterpret_cast<uintptr_t>( + static_cast<Value *>(Val.Dividend)) ^ + reinterpret_cast<uintptr_t>( + static_cast<Value *>(Val.Divisor))) ^ (unsigned)Val.SignedOp; } }; diff --git a/include/llvm/Transforms/Utils/CodeExtractor.h b/include/llvm/Transforms/Utils/CodeExtractor.h index 9d79ee1633f6..8a1ab796734e 100644 --- a/include/llvm/Transforms/Utils/CodeExtractor.h +++ b/include/llvm/Transforms/Utils/CodeExtractor.h @@ -22,6 +22,7 @@ namespace llvm { +class AllocaInst; class BasicBlock; class BlockFrequency; class BlockFrequencyInfo; @@ -36,6 +37,38 @@ class Module; class Type; class Value; +/// A cache for the CodeExtractor analysis. The operation \ref +/// CodeExtractor::extractCodeRegion is guaranteed not to invalidate this +/// object. This object should conservatively be considered invalid if any +/// other mutating operations on the IR occur. +/// +/// Constructing this object is O(n) in the size of the function. +class CodeExtractorAnalysisCache { + /// The allocas in the function. + SmallVector<AllocaInst *, 16> Allocas; + + /// Base memory addresses of load/store instructions, grouped by block. + DenseMap<BasicBlock *, DenseSet<Value *>> BaseMemAddrs; + + /// Blocks which contain instructions which may have unknown side-effects + /// on memory. + DenseSet<BasicBlock *> SideEffectingBlocks; + + void findSideEffectInfoForBlock(BasicBlock &BB); + +public: + CodeExtractorAnalysisCache(Function &F); + + /// Get the allocas in the function at the time the analysis was created. + /// Note that some of these allocas may no longer be present in the function, + /// due to \ref CodeExtractor::extractCodeRegion. + ArrayRef<AllocaInst *> getAllocas() const { return Allocas; } + + /// Check whether \p BB contains an instruction thought to load from, store + /// to, or otherwise clobber the alloca \p Addr. + bool doesBlockContainClobberOfAddr(BasicBlock &BB, AllocaInst *Addr) const; +}; + /// Utility class for extracting code into a new function. /// /// This utility provides a simple interface for extracting some sequence of @@ -104,13 +137,21 @@ class Value; /// /// Returns zero when called on a CodeExtractor instance where isEligible /// returns false. - Function *extractCodeRegion(); + Function *extractCodeRegion(const CodeExtractorAnalysisCache &CEAC); + + /// Verify that assumption cache isn't stale after a region is extracted. + /// Returns false when verifier finds errors. AssumptionCache is passed as + /// parameter to make this function stateless. + static bool verifyAssumptionCache(const Function& F, AssumptionCache *AC); /// Test whether this code extractor is eligible. /// /// Based on the blocks used when constructing the code extractor, /// determine whether it is eligible for extraction. - bool isEligible() const { return !Blocks.empty(); } + /// + /// Checks that varargs handling (with vastart and vaend) is only done in + /// the outlined blocks. + bool isEligible() const; /// Compute the set of input values and output values for the code. /// @@ -127,7 +168,9 @@ class Value; /// region. /// /// Returns true if it is safe to do the code motion. - bool isLegalToShrinkwrapLifetimeMarkers(Instruction *AllocaAddr) const; + bool + isLegalToShrinkwrapLifetimeMarkers(const CodeExtractorAnalysisCache &CEAC, + Instruction *AllocaAddr) const; /// Find the set of allocas whose life ranges are contained within the /// outlined region. @@ -137,7 +180,8 @@ class Value; /// are used by the lifetime markers are also candidates for shrink- /// wrapping. The instructions that need to be sunk are collected in /// 'Allocas'. - void findAllocas(ValueSet &SinkCands, ValueSet &HoistCands, + void findAllocas(const CodeExtractorAnalysisCache &CEAC, + ValueSet &SinkCands, ValueSet &HoistCands, BasicBlock *&ExitBlock) const; /// Find or create a block within the outline region for placing hoisted @@ -158,8 +202,9 @@ class Value; Instruction *LifeEnd = nullptr; }; - LifetimeMarkerInfo getLifetimeMarkers(Instruction *Addr, - BasicBlock *ExitBlock) const; + LifetimeMarkerInfo + getLifetimeMarkers(const CodeExtractorAnalysisCache &CEAC, + Instruction *Addr, BasicBlock *ExitBlock) const; void severSplitPHINodesOfEntry(BasicBlock *&Header); void severSplitPHINodesOfExits(const SmallPtrSetImpl<BasicBlock *> &Exits); diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h index ff516f230979..9fcb2f64d79b 100644 --- a/include/llvm/Transforms/Utils/Local.h +++ b/include/llvm/Transforms/Utils/Local.h @@ -271,6 +271,15 @@ inline unsigned getKnownAlignment(Value *V, const DataLayout &DL, return getOrEnforceKnownAlignment(V, 0, DL, CxtI, AC, DT); } +/// Create a call that matches the invoke \p II in terms of arguments, +/// attributes, debug information, etc. The call is not placed in a block and it +/// will not have a name. The invoke instruction is not removed, nor are the +/// uses replaced by the new call. +CallInst *createCallMatchingInvoke(InvokeInst *II); + +/// This function converts the specified invoek into a normall call. +void changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr); + ///===---------------------------------------------------------------------===// /// Dbg Intrinsic utilities /// @@ -403,8 +412,7 @@ void removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU = nullptr); /// Remove all blocks that can not be reached from the function's entry. /// /// Returns true if any basic block was removed. -bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI = nullptr, - DomTreeUpdater *DTU = nullptr, +bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr, MemorySSAUpdater *MSSAU = nullptr); /// Combine the metadata of two instructions so that K can replace J. Some @@ -424,6 +432,10 @@ void combineMetadata(Instruction *K, const Instruction *J, void combineMetadataForCSE(Instruction *K, const Instruction *J, bool DoesKMove); +/// Copy the metadata from the source instruction to the destination (the +/// replacement for the source instruction). +void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source); + /// Patch the replacement so that it is not more restrictive than the value /// being replaced. It assumes that the replacement does not get moved from /// its original position. diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h index 68bdded5cf93..d32f08717e9b 100644 --- a/include/llvm/Transforms/Utils/LoopUtils.h +++ b/include/llvm/Transforms/Utils/LoopUtils.h @@ -215,6 +215,9 @@ makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef<StringRef> FollowupAttrs, /// Look for the loop attribute that disables all transformation heuristic. bool hasDisableAllTransformsHint(const Loop *L); +/// Look for the loop attribute that disables the LICM transformation heuristics. +bool hasDisableLICMTransformsHint(const Loop *L); + /// The mode sets how eager a transformation should be applied. enum TransformationMode { /// The pass can use heuristics to determine whether a transformation should @@ -252,6 +255,8 @@ TransformationMode hasLICMVersioningTransformation(Loop *L); /// @} /// Set input string into loop metadata by keeping other values intact. +/// If the string is already in loop metadata update value if it is +/// different. void addStringMetadataToLoop(Loop *TheLoop, const char *MDString, unsigned V = 0); diff --git a/include/llvm/Transforms/Utils/MisExpect.h b/include/llvm/Transforms/Utils/MisExpect.h new file mode 100644 index 000000000000..1dbe8cb95936 --- /dev/null +++ b/include/llvm/Transforms/Utils/MisExpect.h @@ -0,0 +1,43 @@ +//===--- MisExpect.h - Check the use of llvm.expect with PGO data ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This contains code to emit warnings for potentially incorrect usage of the +// llvm.expect intrinsic. This utility extracts the threshold values from +// metadata associated with the instrumented Branch or Switch instruction. The +// threshold values are then used to determine if a warning should be emmited. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LLVMContext.h" + +namespace llvm { +namespace misexpect { + +/// verifyMisExpect - compares PGO counters to the thresholds used for +/// llvm.expect and warns if the PGO counters are outside of the expected +/// range. +/// \param I The Instruction being checked +/// \param Weights A vector of profile weights for each target block +/// \param Ctx The current LLVM context +void verifyMisExpect(llvm::Instruction *I, + const llvm::SmallVector<uint32_t, 4> &Weights, + llvm::LLVMContext &Ctx); + +/// checkClangInstrumentation - verify if llvm.expect matches PGO profile +/// This function checks the frontend instrumentation in the backend when +/// lowering llvm.expect intrinsics. It checks for existing metadata, and +/// then validates the use of llvm.expect against the assigned branch weights. +// +/// \param I the Instruction being checked +void checkFrontendInstrumentation(Instruction &I); + +} // namespace misexpect +} // namespace llvm diff --git a/include/llvm/Transforms/Utils/PredicateInfo.h b/include/llvm/Transforms/Utils/PredicateInfo.h index da4a5dcc28c0..7c7a8eb04a2c 100644 --- a/include/llvm/Transforms/Utils/PredicateInfo.h +++ b/include/llvm/Transforms/Utils/PredicateInfo.h @@ -229,10 +229,10 @@ protected: private: void buildPredicateInfo(); - void processAssume(IntrinsicInst *, BasicBlock *, SmallPtrSetImpl<Value *> &); - void processBranch(BranchInst *, BasicBlock *, SmallPtrSetImpl<Value *> &); - void processSwitch(SwitchInst *, BasicBlock *, SmallPtrSetImpl<Value *> &); - void renameUses(SmallPtrSetImpl<Value *> &); + void processAssume(IntrinsicInst *, BasicBlock *, SmallVectorImpl<Value *> &); + void processBranch(BranchInst *, BasicBlock *, SmallVectorImpl<Value *> &); + void processSwitch(SwitchInst *, BasicBlock *, SmallVectorImpl<Value *> &); + void renameUses(SmallVectorImpl<Value *> &); using ValueDFS = PredicateInfoClasses::ValueDFS; typedef SmallVectorImpl<ValueDFS> ValueDFSStack; void convertUsesToDFSOrdered(Value *, SmallVectorImpl<ValueDFS> &); @@ -240,7 +240,7 @@ private: bool stackIsInScope(const ValueDFSStack &, const ValueDFS &) const; void popStackUntilDFSScope(ValueDFSStack &, const ValueDFS &); ValueInfo &getOrCreateValueInfo(Value *); - void addInfoFor(SmallPtrSetImpl<Value *> &OpsToRename, Value *Op, + void addInfoFor(SmallVectorImpl<Value *> &OpsToRename, Value *Op, PredicateBase *PB); const ValueInfo &getValueInfo(Value *) const; Function &F; diff --git a/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/include/llvm/Transforms/Utils/SimplifyLibCalls.h index 2572094ddac8..88c2ef787ad8 100644 --- a/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -126,6 +126,12 @@ private: /// Erase an instruction from its parent with our eraser. void eraseFromParent(Instruction *I); + /// Replace an instruction with a value and erase it from its parent. + void substituteInParent(Instruction *I, Value *With) { + replaceAllUsesWith(I, With); + eraseFromParent(I); + } + Value *foldMallocMemset(CallInst *Memset, IRBuilder<> &B); public: @@ -154,6 +160,7 @@ private: Value *optimizeStrRChr(CallInst *CI, IRBuilder<> &B); Value *optimizeStrCmp(CallInst *CI, IRBuilder<> &B); Value *optimizeStrNCmp(CallInst *CI, IRBuilder<> &B); + Value *optimizeStrNDup(CallInst *CI, IRBuilder<> &B); Value *optimizeStrCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeStpCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeStrNCpy(CallInst *CI, IRBuilder<> &B); @@ -164,14 +171,17 @@ private: Value *optimizeStrCSpn(CallInst *CI, IRBuilder<> &B); Value *optimizeStrStr(CallInst *CI, IRBuilder<> &B); Value *optimizeMemChr(CallInst *CI, IRBuilder<> &B); + Value *optimizeMemRChr(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCmp(CallInst *CI, IRBuilder<> &B); Value *optimizeBCmp(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCmpBCmpCommon(CallInst *CI, IRBuilder<> &B); + Value *optimizeMemPCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemMove(CallInst *CI, IRBuilder<> &B); Value *optimizeMemSet(CallInst *CI, IRBuilder<> &B); Value *optimizeRealloc(CallInst *CI, IRBuilder<> &B); Value *optimizeWcslen(CallInst *CI, IRBuilder<> &B); + Value *optimizeBCopy(CallInst *CI, IRBuilder<> &B); // Wrapper for all String/Memory Library Call Optimizations Value *optimizeStringMemoryLibCall(CallInst *CI, IRBuilder<> &B); diff --git a/include/llvm/Transforms/Utils/UnrollLoop.h b/include/llvm/Transforms/Utils/UnrollLoop.h index 593ca26feb98..02b81b4b7ee2 100644 --- a/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/include/llvm/Transforms/Utils/UnrollLoop.h @@ -114,8 +114,8 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues, OptimizationRemarkEmitter *ORE, unsigned &TripCount, - unsigned MaxTripCount, unsigned &TripMultiple, - unsigned LoopSize, + unsigned MaxTripCount, bool MaxOrZero, + unsigned &TripMultiple, unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound); @@ -132,7 +132,9 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences( BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, int OptLevel, Optional<unsigned> UserThreshold, Optional<unsigned> UserCount, Optional<bool> UserAllowPartial, Optional<bool> UserRuntime, - Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling); + Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling, + Optional<bool> UserAllowProfileBasedPeeling, + Optional<unsigned> UserFullUnrollMaxCount); unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls, bool &NotDuplicatable, bool &Convergent, diff --git a/include/llvm/Transforms/Utils/ValueMapper.h b/include/llvm/Transforms/Utils/ValueMapper.h index 1952a210291e..ff5bfc609586 100644 --- a/include/llvm/Transforms/Utils/ValueMapper.h +++ b/include/llvm/Transforms/Utils/ValueMapper.h @@ -22,7 +22,7 @@ namespace llvm { class Constant; class Function; -class GlobalAlias; +class GlobalIndirectSymbol; class GlobalVariable; class Instruction; class MDNode; @@ -120,7 +120,7 @@ inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { /// instance: /// - \a scheduleMapGlobalInitializer() /// - \a scheduleMapAppendingVariable() -/// - \a scheduleMapGlobalAliasee() +/// - \a scheduleMapGlobalIndirectSymbol() /// - \a scheduleRemapFunction() /// /// Sometimes a callback needs a different mapping context. Such a context can @@ -180,8 +180,9 @@ public: bool IsOldCtorDtor, ArrayRef<Constant *> NewMembers, unsigned MappingContextID = 0); - void scheduleMapGlobalAliasee(GlobalAlias &GA, Constant &Aliasee, - unsigned MappingContextID = 0); + void scheduleMapGlobalIndirectSymbol(GlobalIndirectSymbol &GIS, + Constant &Target, + unsigned MappingContextID = 0); void scheduleRemapFunction(Function &F, unsigned MappingContextID = 0); }; diff --git a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index b144006e2628..d1e7acc877bf 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -33,18 +33,6 @@ namespace llvm { -/// Create an analysis remark that explains why vectorization failed -/// -/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p -/// RemarkName is the identifier for the remark. If \p I is passed it is an -/// instruction that prevents vectorization. Otherwise \p TheLoop is used for -/// the location of the remark. \return the remark object that can be -/// streamed to. -OptimizationRemarkAnalysis createLVMissedAnalysis(const char *PassName, - StringRef RemarkName, - Loop *TheLoop, - Instruction *I = nullptr); - /// Utility class for getting and setting loop vectorizer hints in the form /// of loop metadata. /// This class keeps a number of loop annotations locally (as member variables) @@ -55,7 +43,8 @@ OptimizationRemarkAnalysis createLVMissedAnalysis(const char *PassName, /// for example 'force', means a decision has been made. So, we need to be /// careful NOT to add them if the user hasn't specifically asked so. class LoopVectorizeHints { - enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED }; + enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED, + HK_PREDICATE }; /// Hint - associates name and validation with the hint value. struct Hint { @@ -81,6 +70,9 @@ class LoopVectorizeHints { /// Already Vectorized Hint IsVectorized; + /// Vector Predicate + Hint Predicate; + /// Return the loop metadata prefix. static StringRef Prefix() { return "llvm.loop."; } @@ -109,6 +101,7 @@ public: unsigned getWidth() const { return Width.Value; } unsigned getInterleave() const { return Interleave.Value; } unsigned getIsVectorized() const { return IsVectorized.Value; } + unsigned getPredicate() const { return Predicate.Value; } enum ForceKind getForce() const { if ((ForceKind)Force.Value == FK_Undefined && hasDisableAllTransformsHint(TheLoop)) @@ -235,8 +228,8 @@ public: bool canVectorize(bool UseVPlanNativePath); /// Return true if we can vectorize this loop while folding its tail by - /// masking. - bool canFoldTailByMasking(); + /// masking, and mark all respective loads/stores for masking. + bool prepareToFoldTailByMasking(); /// Returns the primary induction variable. PHINode *getPrimaryInduction() { return PrimaryInduction; } @@ -362,9 +355,16 @@ private: bool canVectorizeOuterLoop(); /// Return true if all of the instructions in the block can be speculatively - /// executed. \p SafePtrs is a list of addresses that are known to be legal - /// and we know that we can read from them without segfault. - bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs); + /// executed, and record the loads/stores that require masking. If's that + /// guard loads can be ignored under "assume safety" unless \p PreserveGuards + /// is true. This can happen when we introduces guards for which the original + /// "unguarded-loads are safe" assumption does not hold. For example, the + /// vectorizer's fold-tail transformation changes the loop to execute beyond + /// its original trip-count, under a proper guard, which should be preserved. + /// \p SafePtrs is a list of addresses that are known to be legal and we know + /// that we can read from them without segfault. + bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs, + bool PreserveGuards = false); /// Updates the vectorization state by adding \p Phi to the inductions list. /// This can set \p Phi as the main induction of the loop if \p Phi is a @@ -382,14 +382,6 @@ private: return LAI ? &LAI->getSymbolicStrides() : nullptr; } - /// Reports a vectorization illegality: print \p DebugMsg for debugging - /// purposes along with the corresponding optimization remark \p RemarkName. - /// If \p I is passed it is an instruction that prevents vectorization. - /// Otherwise the loop is used for the location of the remark. - void reportVectorizationFailure(const StringRef DebugMsg, - const StringRef OREMsg, const StringRef ORETag, - Instruction *I = nullptr) const; - /// The loop that we evaluate. Loop *TheLoop; @@ -452,8 +444,8 @@ private: /// Holds the widest induction type encountered. Type *WidestIndTy = nullptr; - /// Allowed outside users. This holds the induction and reduction - /// vars which can be accessed from outside the loop. + /// Allowed outside users. This holds the variables that can be accessed from + /// outside the loop. SmallPtrSet<Value *, 4> AllowedExit; /// Can we assume the absence of NaNs. diff --git a/include/llvm/Transforms/Vectorize/LoopVectorize.h b/include/llvm/Transforms/Vectorize/LoopVectorize.h index d1ec06afb02a..d824e2903ef3 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -155,6 +155,14 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { bool processLoop(Loop *L); }; +/// Reports a vectorization failure: print \p DebugMsg for debugging +/// purposes along with the corresponding optimization remark \p RemarkName. +/// If \p I is passed, it is an instruction that prevents vectorization. +/// Otherwise, the loop \p TheLoop is used for the location of the remark. +void reportVectorizationFailure(const StringRef DebugMsg, + const StringRef OREMsg, const StringRef ORETag, + OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I = nullptr); + } // end namespace llvm #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H diff --git a/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/include/llvm/Transforms/Vectorize/SLPVectorizer.h index ac6afb761d4d..32ccc8a46380 100644 --- a/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -24,7 +24,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/ValueHandle.h" namespace llvm { @@ -60,8 +59,8 @@ extern cl::opt<bool> RunSLPVectorization; struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> { using StoreList = SmallVector<StoreInst *, 8>; using StoreListMap = MapVector<Value *, StoreList>; - using WeakTrackingVHList = SmallVector<WeakTrackingVH, 8>; - using WeakTrackingVHListMap = MapVector<Value *, WeakTrackingVHList>; + using GEPList = SmallVector<GetElementPtrInst *, 8>; + using GEPListMap = MapVector<Value *, GEPList>; ScalarEvolution *SE = nullptr; TargetTransformInfo *TTI = nullptr; @@ -131,7 +130,7 @@ private: /// Tries to vectorize constructs started from CmpInst, InsertValueInst or /// InsertElementInst instructions. - bool vectorizeSimpleInstructions(SmallVectorImpl<WeakVH> &Instructions, + bool vectorizeSimpleInstructions(SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, slpvectorizer::BoUpSLP &R); /// Scan the basic block and look for patterns that are likely to start @@ -147,7 +146,7 @@ private: StoreListMap Stores; /// The getelementptr instructions in a basic block organized by base pointer. - WeakTrackingVHListMap GEPs; + GEPListMap GEPs; }; } // end namespace llvm diff --git a/include/llvm/XRay/FDRRecordProducer.h b/include/llvm/XRay/FDRRecordProducer.h index b530a85bc7e1..043d91568f4e 100644 --- a/include/llvm/XRay/FDRRecordProducer.h +++ b/include/llvm/XRay/FDRRecordProducer.h @@ -27,7 +27,7 @@ public: class FileBasedRecordProducer : public RecordProducer { const XRayFileHeader &Header; DataExtractor &E; - uint32_t &OffsetPtr; + uint64_t &OffsetPtr; uint32_t CurrentBufferBytes = 0; // Helper function which gets the next record by speculatively reading through @@ -36,7 +36,7 @@ class FileBasedRecordProducer : public RecordProducer { public: FileBasedRecordProducer(const XRayFileHeader &FH, DataExtractor &DE, - uint32_t &OP) + uint64_t &OP) : Header(FH), E(DE), OffsetPtr(OP) {} /// This producer encapsulates the logic for loading a File-backed diff --git a/include/llvm/XRay/FDRRecords.h b/include/llvm/XRay/FDRRecords.h index a8ce74bd88fb..e3e16f71e2fe 100644 --- a/include/llvm/XRay/FDRRecords.h +++ b/include/llvm/XRay/FDRRecords.h @@ -417,16 +417,16 @@ public: class RecordInitializer : public RecordVisitor { DataExtractor &E; - uint32_t &OffsetPtr; + uint64_t &OffsetPtr; uint16_t Version; public: static constexpr uint16_t DefaultVersion = 5u; - explicit RecordInitializer(DataExtractor &DE, uint32_t &OP, uint16_t V) + explicit RecordInitializer(DataExtractor &DE, uint64_t &OP, uint16_t V) : RecordVisitor(), E(DE), OffsetPtr(OP), Version(V) {} - explicit RecordInitializer(DataExtractor &DE, uint32_t &OP) + explicit RecordInitializer(DataExtractor &DE, uint64_t &OP) : RecordInitializer(DE, OP, DefaultVersion) {} Error visit(BufferExtents &) override; diff --git a/include/llvm/XRay/FileHeaderReader.h b/include/llvm/XRay/FileHeaderReader.h index 1c9681cfd9af..30878f3e99e8 100644 --- a/include/llvm/XRay/FileHeaderReader.h +++ b/include/llvm/XRay/FileHeaderReader.h @@ -24,7 +24,7 @@ namespace xray { /// Convenience function for loading the file header given a data extractor at a /// specified offset. Expected<XRayFileHeader> readBinaryFormatHeader(DataExtractor &HeaderExtractor, - uint32_t &OffsetPtr); + uint64_t &OffsetPtr); } // namespace xray } // namespace llvm diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap index 9c4668e1473c..ecb3b37004fd 100644 --- a/include/llvm/module.modulemap +++ b/include/llvm/module.modulemap @@ -253,6 +253,7 @@ module LLVM_IR { textual header "IR/DebugInfoFlags.def" textual header "IR/Instruction.def" textual header "IR/Metadata.def" + textual header "IR/FixedMetadataKinds.def" textual header "IR/Value.def" textual header "IR/RuntimeLibcalls.def" } @@ -331,6 +332,7 @@ module LLVM_TableGen { module LLVM_Transforms { requires cplusplus umbrella "Transforms" + module * { export * } } |