diff options
Diffstat (limited to 'llvm/include')
875 files changed, 26480 insertions, 13381 deletions
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index 09d80841fa5d..2abc29851cd9 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -549,6 +549,13 @@ LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C); void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard); /** + * Set whether the given context is in opaque pointer mode. + * + * @see LLVMContext::setOpaquePointers() + */ +void LLVMContextSetOpaquePointers(LLVMContextRef C, LLVMBool OpaquePointers); + +/** * Destroy a context instance. * * This should be called for every call to LLVMContextCreate() or memory @@ -1391,9 +1398,9 @@ LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy); */ /** - * Obtain the type of elements within a sequential type. + * Obtain the element type of an array or vector type. * - * This works on array, vector, and pointer types. + * This currently also works for pointer types, but this usage is deprecated. * * @see llvm::SequentialType::getElementType() */ @@ -1443,6 +1450,22 @@ unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy); LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType, unsigned AddressSpace); /** + * Determine whether a pointer is opaque. + * + * True if this is an instance of an opaque PointerType. + * + * @see llvm::Type::isOpaquePointerTy() + */ +LLVMBool LLVMPointerTypeIsOpaque(LLVMTypeRef Ty); + +/** + * Create an opaque pointer type in a context. + * + * @see llvm::PointerType::get() + */ +LLVMTypeRef LLVMPointerTypeInContext(LLVMContextRef C, unsigned AddressSpace); + +/** * Obtain the address space of a pointer type. * * This only works on types that represent pointers. @@ -2089,11 +2112,23 @@ LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy, unsigned Count); /** + * Get element of a constant aggregate (struct, array or vector) at the + * specified index. Returns null if the index is out of range, or it's not + * possible to determine the element (e.g., because the constant is a + * constant expression.) + * + * @see llvm::Constant::getAggregateElement() + */ +LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx); + +/** * Get an element at specified index as a constant. * * @see ConstantDataSequential::getElementAsConstant() */ -LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef C, unsigned idx); +LLVM_ATTRIBUTE_C_DEPRECATED( + LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef C, unsigned idx), + "Use LLVMGetAggregateElement instead"); /** * Create a ConstantVector from values. @@ -2203,8 +2238,6 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant, LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant, LLVMValueRef VectorBConstant, LLVMValueRef MaskConstant); -LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList, - unsigned NumIdx); LLVMValueRef LLVMConstInsertValue(LLVMValueRef AggConstant, LLVMValueRef ElementValueConstant, unsigned *IdxList, unsigned NumIdx); @@ -3978,6 +4011,9 @@ LLVMValueRef LLVMBuildFPCast(LLVMBuilderRef, LLVMValueRef Val, LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/ LLVMTypeRef DestTy, const char *Name); +LLVMOpcode LLVMGetCastOpcode(LLVMValueRef Src, LLVMBool SrcIsSigned, + LLVMTypeRef DestTy, LLVMBool DestIsSigned); + /* Comparisons */ LLVMValueRef LLVMBuildICmp(LLVMBuilderRef, LLVMIntPredicate Op, LLVMValueRef LHS, LLVMValueRef RHS, diff --git a/llvm/include/llvm-c/DisassemblerTypes.h b/llvm/include/llvm-c/DisassemblerTypes.h index 53baaef11033..6999a350ec91 100644 --- a/llvm/include/llvm-c/DisassemblerTypes.h +++ b/llvm/include/llvm-c/DisassemblerTypes.h @@ -38,15 +38,15 @@ typedef void *LLVMDisasmContextRef; * one operand with symbolic information. To determine the symbolic operand * information for each operand, the bytes for the specific operand in the * instruction are specified by the Offset parameter and its byte widith is the - * size parameter. For instructions sets with fixed widths and one symbolic - * operand per instruction, the Offset parameter will be zero and Size parameter - * will be the instruction width. The information is returned in TagBuf and is - * Triple specific with its specific information defined by the value of - * TagType for that Triple. If symbolic information is returned the function - * returns 1, otherwise it returns 0. + * OpSize parameter. For instructions sets with fixed widths and one symbolic + * operand per instruction, the Offset parameter will be zero and InstSize + * parameter will be the instruction width. The information is returned in + * TagBuf and is Triple specific with its specific information defined by the + * value of TagType for that Triple. If symbolic information is returned the + * function * returns 1, otherwise it returns 0. */ -typedef int (*LLVMOpInfoCallback)(void *DisInfo, uint64_t PC, - uint64_t Offset, uint64_t Size, +typedef int (*LLVMOpInfoCallback)(void *DisInfo, uint64_t PC, uint64_t Offset, + uint64_t OpSize, uint64_t InstSize, int TagType, void *TagBuf); /** diff --git a/llvm/include/llvm-c/Object.h b/llvm/include/llvm-c/Object.h index 9a9596aaa08c..f422c1ad224d 100644 --- a/llvm/include/llvm-c/Object.h +++ b/llvm/include/llvm-c/Object.h @@ -38,21 +38,23 @@ typedef struct LLVMOpaqueSymbolIterator *LLVMSymbolIteratorRef; typedef struct LLVMOpaqueRelocationIterator *LLVMRelocationIteratorRef; typedef enum { - LLVMBinaryTypeArchive, /**< Archive file. */ - LLVMBinaryTypeMachOUniversalBinary, /**< Mach-O Universal Binary file. */ - LLVMBinaryTypeCOFFImportFile, /**< COFF Import file. */ - LLVMBinaryTypeIR, /**< LLVM IR. */ - LLVMBinaryTypeWinRes, /**< Windows resource (.res) file. */ - LLVMBinaryTypeCOFF, /**< COFF Object file. */ - LLVMBinaryTypeELF32L, /**< ELF 32-bit, little endian. */ - LLVMBinaryTypeELF32B, /**< ELF 32-bit, big endian. */ - LLVMBinaryTypeELF64L, /**< ELF 64-bit, little endian. */ - LLVMBinaryTypeELF64B, /**< ELF 64-bit, big endian. */ - LLVMBinaryTypeMachO32L, /**< MachO 32-bit, little endian. */ - LLVMBinaryTypeMachO32B, /**< MachO 32-bit, big endian. */ - LLVMBinaryTypeMachO64L, /**< MachO 64-bit, little endian. */ - LLVMBinaryTypeMachO64B, /**< MachO 64-bit, big endian. */ - LLVMBinaryTypeWasm, /**< Web Assembly. */ + LLVMBinaryTypeArchive, /**< Archive file. */ + LLVMBinaryTypeMachOUniversalBinary, /**< Mach-O Universal Binary file. */ + LLVMBinaryTypeCOFFImportFile, /**< COFF Import file. */ + LLVMBinaryTypeIR, /**< LLVM IR. */ + LLVMBinaryTypeWinRes, /**< Windows resource (.res) file. */ + LLVMBinaryTypeCOFF, /**< COFF Object file. */ + LLVMBinaryTypeELF32L, /**< ELF 32-bit, little endian. */ + LLVMBinaryTypeELF32B, /**< ELF 32-bit, big endian. */ + LLVMBinaryTypeELF64L, /**< ELF 64-bit, little endian. */ + LLVMBinaryTypeELF64B, /**< ELF 64-bit, big endian. */ + LLVMBinaryTypeMachO32L, /**< MachO 32-bit, little endian. */ + LLVMBinaryTypeMachO32B, /**< MachO 32-bit, big endian. */ + LLVMBinaryTypeMachO64L, /**< MachO 64-bit, little endian. */ + LLVMBinaryTypeMachO64B, /**< MachO 64-bit, big endian. */ + LLVMBinaryTypeWasm, /**< Web Assembly. */ + LLVMBinaryTypeOffload, /**< Offloading fatbinary. */ + } LLVMBinaryType; /** diff --git a/llvm/include/llvm-c/Orc.h b/llvm/include/llvm-c/Orc.h index e2f30b7cdf45..0dcfb06865aa 100644 --- a/llvm/include/llvm-c/Orc.h +++ b/llvm/include/llvm-c/Orc.h @@ -54,6 +54,7 @@ typedef uint64_t LLVMOrcExecutorAddress; * Represents generic linkage flags for a symbol definition. */ typedef enum { + LLVMJITSymbolGenericFlagsNone = 0, LLVMJITSymbolGenericFlagsExported = 1U << 0, LLVMJITSymbolGenericFlagsWeak = 1U << 1, LLVMJITSymbolGenericFlagsCallable = 1U << 2, @@ -122,13 +123,13 @@ typedef LLVMOrcCSymbolFlagsMapPair *LLVMOrcCSymbolFlagsMapPairs; typedef struct { LLVMOrcSymbolStringPoolEntryRef Name; LLVMJITEvaluatedSymbol Sym; -} LLVMJITCSymbolMapPair; +} LLVMOrcCSymbolMapPair; /** * Represents a list of (SymbolStringPtr, JITEvaluatedSymbol) pairs that can be * used to construct a SymbolMap. */ -typedef LLVMJITCSymbolMapPair *LLVMOrcCSymbolMapPairs; +typedef LLVMOrcCSymbolMapPair *LLVMOrcCSymbolMapPairs; /** * Represents a SymbolAliasMapEntry @@ -203,6 +204,22 @@ typedef enum { } LLVMOrcJITDylibLookupFlags; /** + * An element type for a JITDylib search order. + */ +typedef struct { + LLVMOrcJITDylibRef JD; + LLVMOrcJITDylibLookupFlags JDLookupFlags; +} LLVMOrcCJITDylibSearchOrderElement; + +/** + * A JITDylib search order. + * + * The list is terminated with an element containing a null pointer for the JD + * field. + */ +typedef LLVMOrcCJITDylibSearchOrderElement *LLVMOrcCJITDylibSearchOrder; + +/** * Symbol lookup flags for lookup sets. This should be kept in sync with * llvm::orc::SymbolLookupFlags. */ @@ -341,6 +358,14 @@ typedef LLVMErrorRef (*LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction)( LLVMOrcCLookupSet LookupSet, size_t LookupSetSize); /** + * Disposer for a custom generator. + * + * Will be called by ORC when the JITDylib that the generator is attached to + * is destroyed. + */ +typedef void (*LLVMOrcDisposeCAPIDefinitionGeneratorFunction)(void *Ctx); + +/** * Predicate function for SymbolStringPoolEntries. */ typedef int (*LLVMOrcSymbolPredicate)(void *Ctx, @@ -495,6 +520,58 @@ LLVMOrcSymbolStringPoolEntryRef LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name); /** + * Callback type for ExecutionSession lookups. + * + * If Err is LLVMErrorSuccess then Result will contain a pointer to a + * list of ( SymbolStringPtr, JITEvaluatedSymbol ) pairs of length NumPairs. + * + * If Err is a failure value then Result and Ctx are undefined and should + * not be accessed. The Callback is responsible for handling the error + * value (e.g. by calling LLVMGetErrorMessage + LLVMDisposeErrorMessage). + * + * The caller retains ownership of the Result array and will release all + * contained symbol names. Clients are responsible for retaining any symbol + * names that they wish to hold after the function returns. + */ +typedef void (*LLVMOrcExecutionSessionLookupHandleResultFunction)( + LLVMErrorRef Err, LLVMOrcCSymbolMapPairs Result, size_t NumPairs, + void *Ctx); + +/** + * Look up symbols in an execution session. + * + * This is a wrapper around the general ExecutionSession::lookup function. + * + * The SearchOrder argument contains a list of (JITDylibs, JITDylibSearchFlags) + * pairs that describe the search order. The JITDylibs will be searched in the + * given order to try to find the symbols in the Symbols argument. + * + * The Symbols argument should contain a null-terminated array of + * (SymbolStringPtr, SymbolLookupFlags) pairs describing the symbols to be + * searched for. This function takes ownership of the elements of the Symbols + * array. The Name fields of the Symbols elements are taken to have been + * retained by the client for this function. The client should *not* release the + * Name fields, but are still responsible for destroying the array itself. + * + * The HandleResult function will be called once all searched for symbols have + * been found, or an error occurs. The HandleResult function will be passed an + * LLVMErrorRef indicating success or failure, and (on success) a + * null-terminated LLVMOrcCSymbolMapPairs array containing the function result, + * and the Ctx value passed to the lookup function. + * + * The client is fully responsible for managing the lifetime of the Ctx object. + * A common idiom is to allocate the context prior to the lookup and deallocate + * it in the handler. + * + * THIS API IS EXPERIMENTAL AND LIKELY TO CHANGE IN THE NEAR FUTURE! + */ +void LLVMOrcExecutionSessionLookup( + LLVMOrcExecutionSessionRef ES, LLVMOrcLookupKind K, + LLVMOrcCJITDylibSearchOrder SearchOrder, size_t SearchOrderSize, + LLVMOrcCLookupSet Symbols, size_t SymbolsSize, + LLVMOrcExecutionSessionLookupHandleResultFunction HandleResult, void *Ctx); + +/** * Increments the ref-count for a SymbolStringPool entry. */ void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S); @@ -504,6 +581,11 @@ void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S); */ void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S); +/** + * Return the c-string for the given symbol. This string will remain valid until + * the entry is freed (once all LLVMOrcSymbolStringPoolEntryRefs have been + * released). + */ const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S); /** @@ -547,7 +629,7 @@ void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU); * unit. This function takes ownership of the elements of the Syms array. The * Name fields of the array elements are taken to have been retained for this * function. The client should *not* release the elements of the array, but is - * still responsible for destroyingthe array itself. + * still responsible for destroying the array itself. * * The InitSym argument indicates whether or not this MaterializationUnit * contains static initializers. If three are no static initializers (the common @@ -701,7 +783,7 @@ LLVMOrcMaterializationResponsibilityGetRequestedSymbols( */ void LLVMOrcDisposeSymbols(LLVMOrcSymbolStringPoolEntryRef *Symbols); -/* +/** * Notifies the target JITDylib that the given symbols have been resolved. * This will update the given symbols' addresses in the JITDylib, and notify * any pending queries on the given symbols of their resolution. The given @@ -901,9 +983,27 @@ void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD, /** * Create a custom generator. + * + * The F argument will be used to implement the DefinitionGenerator's + * tryToGenerate method (see + * LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction). + * + * Ctx is a context object that will be passed to F. This argument is + * permitted to be null. + * + * Dispose is the disposal function for Ctx. This argument is permitted to be + * null (in which case the client is responsible for the lifetime of Ctx). */ LLVMOrcDefinitionGeneratorRef LLVMOrcCreateCustomCAPIDefinitionGenerator( - LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction F, void *Ctx); + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction F, void *Ctx, + LLVMOrcDisposeCAPIDefinitionGeneratorFunction Dispose); + +/** + * Continue a lookup that was suspended in a generator (see + * LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction). + */ +void LLVMOrcLookupStateContinueLookup(LLVMOrcLookupStateRef S, + LLVMErrorRef Err); /** * Get a DynamicLibrarySearchGenerator that will reflect process symbols into diff --git a/llvm/include/llvm-c/TargetMachine.h b/llvm/include/llvm-c/TargetMachine.h index 23c8c63ff0b4..bfbe1421a356 100644 --- a/llvm/include/llvm-c/TargetMachine.h +++ b/llvm/include/llvm-c/TargetMachine.h @@ -136,7 +136,9 @@ void LLVMSetTargetMachineAsmVerbosity(LLVMTargetMachineRef T, wraps several c++ only classes (among them a file stream). Returns any error in ErrorMessage. Use LLVMDisposeMessage to dispose the message. */ LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T, LLVMModuleRef M, - char *Filename, LLVMCodeGenFileType codegen, char **ErrorMessage); + const char *Filename, + LLVMCodeGenFileType codegen, + char **ErrorMessage); /** Compile the LLVM IR stored in \p M and store the result in \p OutMemBuf. */ LLVMBool LLVMTargetMachineEmitToMemoryBuffer(LLVMTargetMachineRef T, LLVMModuleRef M, diff --git a/llvm/include/llvm-c/Transforms/Coroutines.h b/llvm/include/llvm-c/Transforms/Coroutines.h deleted file mode 100644 index 03b6822033c9..000000000000 --- a/llvm/include/llvm-c/Transforms/Coroutines.h +++ /dev/null @@ -1,56 +0,0 @@ -/*===-- Coroutines.h - Coroutines Library C Interface -----------*- C++ -*-===*\ -|* *| -|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| -|* Exceptions. *| -|* See https://llvm.org/LICENSE.txt for license information. *| -|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| -|* *| -|*===----------------------------------------------------------------------===*| -|* *| -|* This header declares the C interface to libLLVMCoroutines.a, which *| -|* implements various scalar transformations of the LLVM IR. *| -|* *| -|* Many exotic languages can interoperate with C code but have a harder time *| -|* with C++ due to name mangling. So in addition to C, this interface enables *| -|* tools written in such languages. *| -|* *| -\*===----------------------------------------------------------------------===*/ - -#ifndef LLVM_C_TRANSFORMS_COROUTINES_H -#define LLVM_C_TRANSFORMS_COROUTINES_H - -#include "llvm-c/ExternC.h" -#include "llvm-c/Types.h" -#include "llvm-c/Transforms/PassManagerBuilder.h" - -LLVM_C_EXTERN_C_BEGIN - -/** - * @defgroup LLVMCTransformsCoroutines Coroutine transformations - * @ingroup LLVMCTransforms - * - * @{ - */ - -/** See llvm::createCoroEarlyLegacyPass function. */ -void LLVMAddCoroEarlyPass(LLVMPassManagerRef PM); - -/** See llvm::createCoroSplitLegacyPass function. */ -void LLVMAddCoroSplitPass(LLVMPassManagerRef PM); - -/** See llvm::createCoroElideLegacyPass function. */ -void LLVMAddCoroElidePass(LLVMPassManagerRef PM); - -/** See llvm::createCoroCleanupLegacyPass function. */ -void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM); - -/** See llvm::addCoroutinePassesToExtensionPoints. */ -void LLVMPassManagerBuilderAddCoroutinePassesToExtensionPoints(LLVMPassManagerBuilderRef PMB); - -/** - * @} - */ - -LLVM_C_EXTERN_C_END - -#endif diff --git a/llvm/include/llvm-c/Transforms/IPO.h b/llvm/include/llvm-c/Transforms/IPO.h index 3f2cadf32366..c806156281bd 100644 --- a/llvm/include/llvm-c/Transforms/IPO.h +++ b/llvm/include/llvm-c/Transforms/IPO.h @@ -27,9 +27,6 @@ LLVM_C_EXTERN_C_BEGIN * @{ */ -/** See llvm::createArgumentPromotionPass function. */ -void LLVMAddArgumentPromotionPass(LLVMPassManagerRef PM); - /** See llvm::createConstantMergePass function. */ void LLVMAddConstantMergePass(LLVMPassManagerRef PM); diff --git a/llvm/include/llvm-c/Transforms/PassManagerBuilder.h b/llvm/include/llvm-c/Transforms/PassManagerBuilder.h index 6e13e18e063b..3ba75440129a 100644 --- a/llvm/include/llvm-c/Transforms/PassManagerBuilder.h +++ b/llvm/include/llvm-c/Transforms/PassManagerBuilder.h @@ -72,12 +72,6 @@ void LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM); -/** See llvm::PassManagerBuilder::populateLTOPassManager. */ -void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB, - LLVMPassManagerRef PM, - LLVMBool Internalize, - LLVMBool RunInliner); - /** * @} */ diff --git a/llvm/include/llvm-c/Transforms/Scalar.h b/llvm/include/llvm-c/Transforms/Scalar.h index ba142508bbe4..1d0944799710 100644 --- a/llvm/include/llvm-c/Transforms/Scalar.h +++ b/llvm/include/llvm-c/Transforms/Scalar.h @@ -94,9 +94,6 @@ void LLVMAddLoopUnrollPass(LLVMPassManagerRef PM); /** See llvm::createLoopUnrollAndJamPass function. */ void LLVMAddLoopUnrollAndJamPass(LLVMPassManagerRef PM); -/** See llvm::createLoopUnswitchPass function. */ -void LLVMAddLoopUnswitchPass(LLVMPassManagerRef PM); - /** See llvm::createLowerAtomicPass function. */ void LLVMAddLowerAtomicPass(LLVMPassManagerRef PM); diff --git a/llvm/include/llvm-c/blake3.h b/llvm/include/llvm-c/blake3.h new file mode 100644 index 000000000000..679477c3aa7f --- /dev/null +++ b/llvm/include/llvm-c/blake3.h @@ -0,0 +1,79 @@ +/*===-- llvm-c/blake3.h - BLAKE3 C Interface ----------------------*- C -*-===*\ +|* *| +|* Released into the public domain with CC0 1.0 *| +|* See 'llvm/lib/Support/BLAKE3/LICENSE' for info. *| +|* SPDX-License-Identifier: CC0-1.0 *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface to LLVM's BLAKE3 implementation. *| +|* Original BLAKE3 C API: https://github.com/BLAKE3-team/BLAKE3/tree/1.3.1/c *| +|* *| +|* Symbols are prefixed with 'llvm' to avoid a potential conflict with *| +|* another BLAKE3 version within the same program. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_BLAKE3_H +#define LLVM_C_BLAKE3_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define LLVM_BLAKE3_VERSION_STRING "1.3.1" +#define LLVM_BLAKE3_KEY_LEN 32 +#define LLVM_BLAKE3_OUT_LEN 32 +#define LLVM_BLAKE3_BLOCK_LEN 64 +#define LLVM_BLAKE3_CHUNK_LEN 1024 +#define LLVM_BLAKE3_MAX_DEPTH 54 + +// This struct is a private implementation detail. It has to be here because +// it's part of llvm_blake3_hasher below. +typedef struct { + uint32_t cv[8]; + uint64_t chunk_counter; + uint8_t buf[LLVM_BLAKE3_BLOCK_LEN]; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} llvm_blake3_chunk_state; + +typedef struct { + uint32_t key[8]; + llvm_blake3_chunk_state chunk; + uint8_t cv_stack_len; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(LLVM_BLAKE3_MAX_DEPTH + 1) * LLVM_BLAKE3_OUT_LEN]; +} llvm_blake3_hasher; + +const char *llvm_blake3_version(void); +void llvm_blake3_hasher_init(llvm_blake3_hasher *self); +void llvm_blake3_hasher_init_keyed(llvm_blake3_hasher *self, + const uint8_t key[LLVM_BLAKE3_KEY_LEN]); +void llvm_blake3_hasher_init_derive_key(llvm_blake3_hasher *self, + const char *context); +void llvm_blake3_hasher_init_derive_key_raw(llvm_blake3_hasher *self, + const void *context, + size_t context_len); +void llvm_blake3_hasher_update(llvm_blake3_hasher *self, const void *input, + size_t input_len); +void llvm_blake3_hasher_finalize(const llvm_blake3_hasher *self, uint8_t *out, + size_t out_len); +void llvm_blake3_hasher_finalize_seek(const llvm_blake3_hasher *self, + uint64_t seek, uint8_t *out, + size_t out_len); +void llvm_blake3_hasher_reset(llvm_blake3_hasher *self); + +#ifdef __cplusplus +} +#endif + +#endif /* LLVM_C_BLAKE3_H */ diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h index 17b57de7b0aa..cdedb6ece992 100644 --- a/llvm/include/llvm/ADT/APFloat.h +++ b/llvm/include/llvm/ADT/APFloat.h @@ -155,7 +155,8 @@ struct APFloatBase { S_IEEEdouble, S_x87DoubleExtended, S_IEEEquad, - S_PPCDoubleDouble + S_PPCDoubleDouble, + S_MaxSemantics = S_PPCDoubleDouble }; static const llvm::fltSemantics &EnumToSemantics(Semantics S); diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h index b1fc85d3c09d..4155cb260a2a 100644 --- a/llvm/include/llvm/ADT/APInt.h +++ b/llvm/include/llvm/ADT/APInt.h @@ -486,7 +486,7 @@ public: return (Ones > 0) && ((Ones + countLeadingZerosSlowCase()) == BitWidth); } - /// Return true if this APInt value contains a sequence of ones with + /// Return true if this APInt value contains a non-empty sequence of ones with /// the remainder zero. bool isShiftedMask() const { if (isSingleWord()) @@ -496,6 +496,23 @@ public: return (Ones + LeadZ + countTrailingZeros()) == BitWidth; } + /// Return true if this APInt value contains a non-empty sequence of ones with + /// the remainder zero. If true, \p MaskIdx will specify the index of the + /// lowest set bit and \p MaskLen is updated to specify the length of the + /// mask, else neither are updated. + bool isShiftedMask(unsigned &MaskIdx, unsigned &MaskLen) const { + if (isSingleWord()) + return isShiftedMask_64(U.VAL, MaskIdx, MaskLen); + unsigned Ones = countPopulationSlowCase(); + unsigned LeadZ = countLeadingZerosSlowCase(); + unsigned TrailZ = countTrailingZerosSlowCase(); + if ((Ones + LeadZ + TrailZ) != BitWidth) + return false; + MaskLen = Ones; + MaskIdx = TrailZ; + return true; + } + /// Compute an APInt containing numBits highbits from this APInt. /// /// Get an APInt with the same BitWidth as this APInt, just zero mask the low @@ -1201,7 +1218,7 @@ public: /// Truncate to new width. /// /// Truncate the APInt to a specified width. It is an error to specify a width - /// that is greater than or equal to the current width. + /// that is greater than the current width. APInt trunc(unsigned width) const; /// Truncate to new width with unsigned saturation. @@ -1221,7 +1238,7 @@ public: /// /// This operation sign extends the APInt to a new width. If the high order /// bit is set, the fill on the left will be done with 1 bits, otherwise zero. - /// It is an error to specify a width that is less than or equal to the + /// It is an error to specify a width that is less than the /// current width. APInt sext(unsigned width) const; @@ -1229,7 +1246,7 @@ public: /// /// This operation zero extends the APInt to a new width. The high order bits /// are filled with 0 bits. It is an error to specify a width that is less - /// than or equal to the current width. + /// than the current width. APInt zext(unsigned width) const; /// Sign extend or truncate to width @@ -1244,24 +1261,6 @@ public: /// extended, truncated, or left alone to make it that width. APInt zextOrTrunc(unsigned width) const; - /// Truncate to width - /// - /// Make this APInt have the bit width given by \p width. The value is - /// truncated or left alone to make it that width. - APInt truncOrSelf(unsigned width) const; - - /// Sign extend or truncate to width - /// - /// Make this APInt have the bit width given by \p width. The value is sign - /// extended, or left alone to make it that width. - APInt sextOrSelf(unsigned width) const; - - /// Zero extend or truncate to width - /// - /// Make this APInt have the bit width given by \p width. The value is zero - /// extended, or left alone to make it that width. - APInt zextOrSelf(unsigned width) const; - /// @} /// \name Bit Manipulation Operators /// @{ @@ -1489,6 +1488,11 @@ public: /// equivalent of the string given by \p str. static unsigned getBitsNeeded(StringRef str, uint8_t radix); + /// Get the bits that are sufficient to represent the string value. This may + /// over estimate the amount of bits required, but it does not require + /// parsing the value in the string. + static unsigned getSufficientBitsNeeded(StringRef Str, uint8_t Radix); + /// The APInt version of the countLeadingZeros functions in /// MathExtras.h. /// @@ -2235,12 +2239,16 @@ Optional<unsigned> GetMostSignificantDifferentBit(const APInt &A, /// Splat/Merge neighboring bits to widen/narrow the bitmask represented /// by \param A to \param NewBitWidth bits. /// +/// MatchAnyBits: (Default) /// e.g. ScaleBitMask(0b0101, 8) -> 0b00110011 /// e.g. ScaleBitMask(0b00011011, 4) -> 0b0111 -/// A.getBitwidth() or NewBitWidth must be a whole multiples of the other. /// -/// TODO: Do we need a mode where all bits must be set when merging down? -APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth); +/// MatchAllBits: +/// e.g. ScaleBitMask(0b0101, 8) -> 0b00110011 +/// e.g. ScaleBitMask(0b00011011, 4) -> 0b0001 +/// A.getBitwidth() or NewBitWidth must be a whole multiples of the other. +APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, + bool MatchAllBits = false); } // namespace APIntOps // See friend declaration above. This additional declaration is required in diff --git a/llvm/include/llvm/ADT/AddressRanges.h b/llvm/include/llvm/ADT/AddressRanges.h new file mode 100644 index 000000000000..1953680d5222 --- /dev/null +++ b/llvm/include/llvm/ADT/AddressRanges.h @@ -0,0 +1,79 @@ +//===- AddressRanges.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_ADDRESSRANGES_H +#define LLVM_ADT_ADDRESSRANGES_H + +#include "llvm/ADT/Optional.h" +#include <cassert> +#include <stdint.h> +#include <vector> + +namespace llvm { + +/// A class that represents an address range. The range is specified using +/// a start and an end address: [Start, End). +class AddressRange { +public: + AddressRange() {} + AddressRange(uint64_t S, uint64_t E) : Start(S), End(E) { + assert(Start <= End); + } + uint64_t start() const { return Start; } + uint64_t end() const { return End; } + uint64_t size() const { return End - Start; } + bool contains(uint64_t Addr) const { return Start <= Addr && Addr < End; } + bool intersects(const AddressRange &R) const { + return Start < R.End && R.Start < End; + } + bool operator==(const AddressRange &R) const { + return Start == R.Start && End == R.End; + } + bool operator!=(const AddressRange &R) const { return !(*this == R); } + bool operator<(const AddressRange &R) const { + return std::make_pair(Start, End) < std::make_pair(R.Start, R.End); + } + +private: + uint64_t Start = 0; + uint64_t End = 0; +}; + +/// The AddressRanges class helps normalize address range collections. +/// This class keeps a sorted vector of AddressRange objects and can perform +/// insertions and searches efficiently. The address ranges are always sorted +/// and never contain any invalid or empty address ranges. Intersecting +/// address ranges are combined during insertion. +class AddressRanges { +protected: + using Collection = std::vector<AddressRange>; + Collection Ranges; + +public: + void clear() { Ranges.clear(); } + bool empty() const { return Ranges.empty(); } + bool contains(uint64_t Addr) const; + bool contains(AddressRange Range) const; + Optional<AddressRange> getRangeThatContains(uint64_t Addr) const; + void insert(AddressRange Range); + void reserve(size_t Capacity) { Ranges.reserve(Capacity); } + size_t size() const { return Ranges.size(); } + bool operator==(const AddressRanges &RHS) const { + return Ranges == RHS.Ranges; + } + const AddressRange &operator[](size_t i) const { + assert(i < Ranges.size()); + return Ranges[i]; + } + Collection::const_iterator begin() const { return Ranges.begin(); } + Collection::const_iterator end() const { return Ranges.end(); } +}; + +} // namespace llvm + +#endif // LLVM_ADT_ADDRESSRANGES_H diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index b6896395dae8..ee35a5686fc4 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -25,6 +25,7 @@ #include <vector> namespace llvm { + template<typename T> class LLVM_NODISCARD MutableArrayRef; /// ArrayRef - Represent a constant reference to an array (0 or more elements /// consecutively in memory), i.e. a start pointer and a length. It allows @@ -175,10 +176,10 @@ namespace llvm { } // copy - Allocate copy in Allocator and return ArrayRef<T> to it. - template <typename Allocator> ArrayRef<T> copy(Allocator &A) { + template <typename Allocator> MutableArrayRef<T> copy(Allocator &A) { T *Buff = A.template Allocate<T>(Length); std::uninitialized_copy(begin(), end(), Buff); - return ArrayRef<T>(Buff, Length); + return MutableArrayRef<T>(Buff, Length); } /// equals - Check for element-wise equality. @@ -539,6 +540,42 @@ namespace llvm { return MutableArrayRef<T>(data, length); } + /// Construct a MutableArrayRef from a SmallVector. + template <typename T> + MutableArrayRef<T> makeMutableArrayRef(SmallVectorImpl<T> &Vec) { + return Vec; + } + + /// Construct a MutableArrayRef from a SmallVector. + template <typename T, unsigned N> + MutableArrayRef<T> makeMutableArrayRef(SmallVector<T, N> &Vec) { + return Vec; + } + + /// Construct a MutableArrayRef from a std::vector. + template<typename T> + MutableArrayRef<T> makeMutableArrayRef(std::vector<T> &Vec) { + return Vec; + } + + /// Construct a MutableArrayRef from a std::array. + template <typename T, std::size_t N> + MutableArrayRef<T> makeMutableArrayRef(std::array<T, N> &Arr) { + return Arr; + } + + /// Construct a MutableArrayRef from a MutableArrayRef (no-op) (const) + template <typename T> + MutableArrayRef<T> makeMutableArrayRef(const MutableArrayRef<T> &Vec) { + return Vec; + } + + /// Construct a MutableArrayRef from a C array. + template<typename T, size_t N> + MutableArrayRef<T> makeMutableArrayRef(T (&Arr)[N]) { + return MutableArrayRef<T>(Arr); + } + /// @} /// @name ArrayRef Comparison Operators /// @{ diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h index 89e5508e08e1..205da1240d44 100644 --- a/llvm/include/llvm/ADT/BitmaskEnum.h +++ b/llvm/include/llvm/ADT/BitmaskEnum.h @@ -77,7 +77,7 @@ namespace BitmaskEnumDetail { /// Get a bitmask with 1s in all places up to the high-order bit of E's largest /// value. -template <typename E> std::underlying_type_t<E> Mask() { +template <typename E> constexpr std::underlying_type_t<E> Mask() { // On overflow, NextPowerOf2 returns zero with the type uint64_t, so // subtracting 1 gives us the mask with all bits set, like we want. return NextPowerOf2(static_cast<std::underlying_type_t<E>>( @@ -87,7 +87,7 @@ template <typename E> std::underlying_type_t<E> Mask() { /// Check that Val is in range for E, and return Val cast to E's underlying /// type. -template <typename E> std::underlying_type_t<E> Underlying(E Val) { +template <typename E> constexpr std::underlying_type_t<E> Underlying(E Val) { auto U = static_cast<std::underlying_type_t<E>>(Val); assert(U >= 0 && "Negative enum values are not allowed."); assert(U <= Mask<E>() && "Enum value too large (or largest val too small?)"); @@ -99,22 +99,22 @@ constexpr unsigned bitWidth(uint64_t Value) { } template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> -E operator~(E Val) { +constexpr E operator~(E Val) { return static_cast<E>(~Underlying(Val) & Mask<E>()); } template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> -E operator|(E LHS, E RHS) { +constexpr E operator|(E LHS, E RHS) { return static_cast<E>(Underlying(LHS) | Underlying(RHS)); } template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> -E operator&(E LHS, E RHS) { +constexpr E operator&(E LHS, E RHS) { return static_cast<E>(Underlying(LHS) & Underlying(RHS)); } template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> -E operator^(E LHS, E RHS) { +constexpr E operator^(E LHS, E RHS) { return static_cast<E>(Underlying(LHS) ^ Underlying(RHS)); } diff --git a/llvm/include/llvm/ADT/BreadthFirstIterator.h b/llvm/include/llvm/ADT/BreadthFirstIterator.h index 1312b5f91e83..807b0a92c48c 100644 --- a/llvm/include/llvm/ADT/BreadthFirstIterator.h +++ b/llvm/include/llvm/ADT/BreadthFirstIterator.h @@ -80,7 +80,7 @@ private: inline void toNext() { Optional<QueueElement> Head = VisitQueue.front(); - QueueElement H = Head.getValue(); + QueueElement H = *Head; NodeRef Node = H.first; Optional<ChildItTy> &ChildIt = H.second; diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h index 7673b66ca42a..c14414c46419 100644 --- a/llvm/include/llvm/ADT/DenseMap.h +++ b/llvm/include/llvm/ADT/DenseMap.h @@ -137,6 +137,7 @@ public: } } assert(NumEntries == 0 && "Node count imbalance!"); + (void)NumEntries; } setNumEntries(0); setNumTombstones(0); diff --git a/llvm/include/llvm/ADT/EpochTracker.h b/llvm/include/llvm/ADT/EpochTracker.h index b06888494466..b46989bc5111 100644 --- a/llvm/include/llvm/ADT/EpochTracker.h +++ b/llvm/include/llvm/ADT/EpochTracker.h @@ -34,10 +34,10 @@ namespace llvm { /// is still valid. /// class DebugEpochBase { - uint64_t Epoch; + uint64_t Epoch = 0; public: - DebugEpochBase() : Epoch(0) {} + DebugEpochBase() = default; /// Calling incrementEpoch invalidates all handles pointing into the /// calling instance. diff --git a/llvm/include/llvm/ADT/EquivalenceClasses.h b/llvm/include/llvm/ADT/EquivalenceClasses.h index f12b683ead2d..4f98b84cf97d 100644 --- a/llvm/include/llvm/ADT/EquivalenceClasses.h +++ b/llvm/include/llvm/ADT/EquivalenceClasses.h @@ -161,7 +161,8 @@ public: // /// iterator* - Provides a way to iterate over all values in the set. - using iterator = typename std::set<ECValue>::const_iterator; + using iterator = + typename std::set<ECValue, ECValueComparator>::const_iterator; iterator begin() const { return TheMapping.begin(); } iterator end() const { return TheMapping.end(); } diff --git a/llvm/include/llvm/ADT/FloatingPointMode.h b/llvm/include/llvm/ADT/FloatingPointMode.h index 9cc69b8a8344..59ccea1f9d44 100644 --- a/llvm/include/llvm/ADT/FloatingPointMode.h +++ b/llvm/include/llvm/ADT/FloatingPointMode.h @@ -7,7 +7,8 @@ //===----------------------------------------------------------------------===// /// /// \file -/// Utilities for dealing with flags related to floating point mode controls. +/// Utilities for dealing with flags related to floating point properties and +/// mode controls. /// //===----------------------------------------------------------------------===/ @@ -193,4 +194,29 @@ void DenormalMode::print(raw_ostream &OS) const { } +/// Floating-point class tests, supported by 'is_fpclass' intrinsic. Actual +/// test may be an OR combination of basic tests. +enum FPClassTest { + fcSNan = 0x0001, + fcQNan = 0x0002, + fcNegInf = 0x0004, + fcNegNormal = 0x0008, + fcNegSubnormal = 0x0010, + fcNegZero = 0x0020, + fcPosZero = 0x0040, + fcPosSubnormal = 0x0080, + fcPosNormal = 0x0100, + fcPosInf = 0x0200, + + fcNan = fcSNan | fcQNan, + fcInf = fcPosInf | fcNegInf, + fcNormal = fcPosNormal | fcNegNormal, + fcSubnormal = fcPosSubnormal | fcNegSubnormal, + fcZero = fcPosZero | fcNegZero, + fcPosFinite = fcPosNormal | fcPosSubnormal | fcPosZero, + fcNegFinite = fcNegNormal | fcNegSubnormal | fcNegZero, + fcFinite = fcPosFinite | fcNegFinite, + fcAllFlags = fcNan | fcInf | fcFinite +}; + #endif // LLVM_ADT_FLOATINGPOINTMODE_H diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h index a8707f0ee81e..ec276d41da80 100644 --- a/llvm/include/llvm/ADT/FoldingSet.h +++ b/llvm/include/llvm/ADT/FoldingSet.h @@ -16,12 +16,14 @@ #ifndef LLVM_ADT_FOLDINGSET_H #define LLVM_ADT_FOLDINGSET_H +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" #include "llvm/Support/Allocator.h" #include <cassert> #include <cstddef> #include <cstdint> +#include <type_traits> #include <utility> namespace llvm { @@ -255,8 +257,8 @@ template<typename T> struct DefaultFoldingSetTrait { /// through template specialization the behavior can be tailored for specific /// types. Combined with the FoldingSetNodeWrapper class, one can add objects /// to FoldingSets that were not originally designed to have that behavior. -template<typename T> struct FoldingSetTrait - : public DefaultFoldingSetTrait<T> {}; +template <typename T, typename Enable = void> +struct FoldingSetTrait : public DefaultFoldingSetTrait<T> {}; /// DefaultContextualFoldingSetTrait - Like DefaultFoldingSetTrait, but /// for ContextualFoldingSets. @@ -293,7 +295,9 @@ public: /// ComputeHash - Compute a strong hash value for this FoldingSetNodeIDRef, /// used to lookup the node in the FoldingSetBase. - unsigned ComputeHash() const; + unsigned ComputeHash() const { + return static_cast<unsigned>(hash_combine_range(Data, Data + Size)); + } bool operator==(FoldingSetNodeIDRef) const; @@ -323,13 +327,33 @@ public: : Bits(Ref.getData(), Ref.getData() + Ref.getSize()) {} /// Add* - Add various data types to Bit data. - void AddPointer(const void *Ptr); - void AddInteger(signed I); - void AddInteger(unsigned I); - void AddInteger(long I); - void AddInteger(unsigned long I); - void AddInteger(long long I); - void AddInteger(unsigned long long I); + void AddPointer(const void *Ptr) { + // Note: this adds pointers to the hash using sizes and endianness that + // depend on the host. It doesn't matter, however, because hashing on + // pointer values is inherently unstable. Nothing should depend on the + // ordering of nodes in the folding set. + static_assert(sizeof(uintptr_t) <= sizeof(unsigned long long), + "unexpected pointer size"); + AddInteger(reinterpret_cast<uintptr_t>(Ptr)); + } + void AddInteger(signed I) { Bits.push_back(I); } + void AddInteger(unsigned I) { Bits.push_back(I); } + void AddInteger(long I) { AddInteger((unsigned long)I); } + void AddInteger(unsigned long I) { + if (sizeof(long) == sizeof(int)) + AddInteger(unsigned(I)); + else if (sizeof(long) == sizeof(long long)) { + AddInteger((unsigned long long)I); + } else { + llvm_unreachable("unexpected sizeof(long)"); + } + } + void AddInteger(long long I) { AddInteger((unsigned long long)I); } + void AddInteger(unsigned long long I) { + AddInteger(unsigned(I)); + AddInteger(unsigned(I >> 32)); + } + void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); } void AddString(StringRef String); void AddNodeID(const FoldingSetNodeID &ID); @@ -343,7 +367,9 @@ public: /// ComputeHash - Compute a strong hash value for this FoldingSetNodeID, used /// to lookup the node in the FoldingSetBase. - unsigned ComputeHash() const; + unsigned ComputeHash() const { + return FoldingSetNodeIDRef(Bits.data(), Bits.size()).ComputeHash(); + } /// operator== - Used to compare two nodes to each other. bool operator==(const FoldingSetNodeID &RHS) const; @@ -803,6 +829,13 @@ struct FoldingSetTrait<std::pair<T1, T2>> { } }; +template <typename T> +struct FoldingSetTrait<T, typename std::enable_if_t<std::is_enum<T>::value>> { + static void Profile(const T &X, FoldingSetNodeID &ID) { + ID.AddInteger(static_cast<typename std::underlying_type_t<T>>(X)); + } +}; + } // end namespace llvm #endif // LLVM_ADT_FOLDINGSET_H diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h index d443f9e21a47..ea2847f8c8ee 100644 --- a/llvm/include/llvm/ADT/GenericCycleImpl.h +++ b/llvm/include/llvm/ADT/GenericCycleImpl.h @@ -66,6 +66,44 @@ void GenericCycle<ContextT>::getExitBlocks( } } +template <typename ContextT> +auto GenericCycle<ContextT>::getCyclePreheader() const -> BlockT * { + BlockT *Predecessor = getCyclePredecessor(); + if (!Predecessor) + return nullptr; + + assert(isReducible() && "Cycle Predecessor must be in a reducible cycle!"); + + if (succ_size(Predecessor) != 1) + return nullptr; + + // Make sure we are allowed to hoist instructions into the predecessor. + if (!Predecessor->isLegalToHoistInto()) + return nullptr; + + return Predecessor; +} + +template <typename ContextT> +auto GenericCycle<ContextT>::getCyclePredecessor() const -> BlockT * { + if (!isReducible()) + return nullptr; + + BlockT *Out = nullptr; + + // Loop over the predecessors of the header node... + BlockT *Header = getHeader(); + for (const auto Pred : predecessors(Header)) { + if (!contains(Pred)) { + if (Out && Out != Pred) + return nullptr; + Out = Pred; + } + } + + return Out; +} + /// \brief Helper class for computing cycle information. template <typename ContextT> class GenericCycleInfoCompute { using BlockT = typename ContextT::BlockT; @@ -267,8 +305,8 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) { DFSTreeStack.emplace_back(TraverseStack.size()); llvm::append_range(TraverseStack, successors(Block)); - LLVM_ATTRIBUTE_UNUSED bool Added = BlockDFSInfo.try_emplace(Block, ++Counter).second; + (void)Added; assert(Added); BlockPreorder.push_back(Block); LLVM_DEBUG(errs() << " preorder number: " << Counter << "\n"); @@ -326,6 +364,19 @@ auto GenericCycleInfo<ContextT>::getCycle(const BlockT *Block) const return nullptr; } +/// \brief get the depth for the cycle which containing a given block. +/// +/// \returns the depth for the innermost cycle containing \p Block or 0 if it is +/// not contained in any cycle. +template <typename ContextT> +unsigned GenericCycleInfo<ContextT>::getCycleDepth(const BlockT *Block) const { + CycleT *Cycle = getCycle(Block); + if (!Cycle) + return 0; + return Cycle->getDepth(); +} + +#ifndef NDEBUG /// \brief Validate the internal consistency of the cycle tree. /// /// Note that this does \em not check that cycles are really cycles in the CFG, @@ -391,6 +442,7 @@ bool GenericCycleInfo<ContextT>::validateTree() const { return true; } +#endif /// \brief Print the cycle info. template <typename ContextT> diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h index d5f9cd9142ac..970664b85715 100644 --- a/llvm/include/llvm/ADT/GenericCycleInfo.h +++ b/llvm/include/llvm/ADT/GenericCycleInfo.h @@ -100,6 +100,10 @@ public: BlockT *getHeader() const { return Entries[0]; } + const SmallVectorImpl<BlockT *> & getEntries() const { + return Entries; + } + /// \brief Return whether \p Block is an entry block of the cycle. bool isEntry(BlockT *Block) const { return is_contained(Entries, Block); } @@ -124,6 +128,16 @@ public: /// branched to. void getExitBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const; + /// Return the preheader block for this cycle. Pre-header is well-defined for + /// reducible cycle in docs/LoopTerminology.rst as: the only one entering + /// block and its only edge is to the entry block. Return null for irreducible + /// cycles. + BlockT *getCyclePreheader() const; + + /// If the cycle has exactly one entry with exactly one predecessor, return + /// it, otherwise return nullptr. + BlockT *getCyclePredecessor() const; + /// Iteration over child cycles. //@{ using const_child_iterator_base = @@ -178,6 +192,7 @@ public: iterator_range<const_entry_iterator> entries() const { return llvm::make_range(Entries.begin(), Entries.end()); } + //@} Printable printEntries(const ContextT &Ctx) const { return Printable([this, &Ctx](raw_ostream &Out) { @@ -238,6 +253,7 @@ public: const ContextT &getSSAContext() const { return Context; } CycleT *getCycle(const BlockT *Block) const; + unsigned getCycleDepth(const BlockT *Block) const; CycleT *getTopLevelParentCycle(const BlockT *Block) const; /// Move \p Child to \p NewParent by manipulating Children vectors. @@ -248,7 +264,9 @@ public: /// Methods for debug and self-test. //@{ +#ifndef NDEBUG bool validateTree() const; +#endif void print(raw_ostream &Out) const; void dump() const { print(dbgs()); } //@} diff --git a/llvm/include/llvm/ADT/IntervalMap.h b/llvm/include/llvm/ADT/IntervalMap.h index 368ed46f98d2..57f02df252c0 100644 --- a/llvm/include/llvm/ADT/IntervalMap.h +++ b/llvm/include/llvm/ADT/IntervalMap.h @@ -106,13 +106,10 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/bit.h" -#include "llvm/Support/AlignOf.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/RecyclingAllocator.h" #include <algorithm> #include <cassert> -#include <cstdint> #include <iterator> #include <new> #include <utility> @@ -969,7 +966,10 @@ public: private: // The root data is either a RootLeaf or a RootBranchData instance. - AlignedCharArrayUnion<RootLeaf, RootBranchData> data; + union { + RootLeaf leaf; + RootBranchData branchData; + }; // Tree height. // 0: Leaves in root. @@ -983,25 +983,22 @@ private: // Allocator used for creating external nodes. Allocator &allocator; - /// Represent data as a node type without breaking aliasing rules. - template <typename T> T &dataAs() const { return *bit_cast<T *>(&data); } - const RootLeaf &rootLeaf() const { assert(!branched() && "Cannot acces leaf data in branched root"); - return dataAs<RootLeaf>(); + return leaf; } RootLeaf &rootLeaf() { assert(!branched() && "Cannot acces leaf data in branched root"); - return dataAs<RootLeaf>(); + return leaf; } - RootBranchData &rootBranchData() const { + const RootBranchData &rootBranchData() const { assert(branched() && "Cannot access branch data in non-branched root"); - return dataAs<RootBranchData>(); + return branchData; } RootBranchData &rootBranchData() { assert(branched() && "Cannot access branch data in non-branched root"); - return dataAs<RootBranchData>(); + return branchData; } const RootBranch &rootBranch() const { return rootBranchData().node; } @@ -1042,11 +1039,20 @@ private: public: explicit IntervalMap(Allocator &a) : height(0), rootSize(0), allocator(a) { - assert((uintptr_t(&data) & (alignof(RootLeaf) - 1)) == 0 && - "Insufficient alignment"); new(&rootLeaf()) RootLeaf(); } + // The default copy/move constructors and assignment operators would perform + // a shallow copy, leading to an incorrect internal state. To prevent + // accidental use, explicitly delete these operators. + // If necessary, implement them to perform a deep copy. + IntervalMap(const IntervalMap &Other) = delete; + IntervalMap(IntervalMap &&Other) = delete; + // Note: these are already implicitly deleted, because RootLeaf (union + // member) has a non-trivial assignment operator (because of std::pair). + IntervalMap &operator=(const IntervalMap &Other) = delete; + IntervalMap &operator=(IntervalMap &&Other) = delete; + ~IntervalMap() { clear(); rootLeaf().~RootLeaf(); diff --git a/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h b/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h index 975535bb5676..e41eb0639ce3 100644 --- a/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h +++ b/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h @@ -84,7 +84,7 @@ protected: #ifndef NDEBUG ~RefCountedBase() { assert(RefCount == 0 && - "Destruction occured when there are still references to this."); + "Destruction occurred when there are still references to this."); } #else // Default the destructor in release builds, A trivial destructor may enable @@ -115,7 +115,7 @@ protected: #ifndef NDEBUG ~ThreadSafeRefCountedBase() { assert(RefCount == 0 && - "Destruction occured when there are still references to this."); + "Destruction occurred when there are still references to this."); } #else // Default the destructor in release builds, A trivial destructor may enable diff --git a/llvm/include/llvm/ADT/Optional.h b/llvm/include/llvm/ADT/Optional.h index e047b0fc6514..d1615d903e98 100644 --- a/llvm/include/llvm/ADT/Optional.h +++ b/llvm/include/llvm/ADT/Optional.h @@ -60,85 +60,96 @@ template <typename T, class OptionalStorage { union { char empty; - T value; + T val; }; - bool hasVal; + bool hasVal = false; public: ~OptionalStorage() { reset(); } - constexpr OptionalStorage() noexcept : empty(), hasVal(false) {} + constexpr OptionalStorage() noexcept : empty() {} constexpr OptionalStorage(OptionalStorage const &other) : OptionalStorage() { - if (other.hasValue()) { - emplace(other.value); + if (other.has_value()) { + emplace(other.val); } } constexpr OptionalStorage(OptionalStorage &&other) : OptionalStorage() { - if (other.hasValue()) { - emplace(std::move(other.value)); + if (other.has_value()) { + emplace(std::move(other.val)); } } template <class... Args> - constexpr explicit OptionalStorage(in_place_t, Args &&... args) - : value(std::forward<Args>(args)...), hasVal(true) {} + constexpr explicit OptionalStorage(in_place_t, Args &&...args) + : val(std::forward<Args>(args)...), hasVal(true) {} void reset() noexcept { if (hasVal) { - value.~T(); + val.~T(); hasVal = false; } } + constexpr bool has_value() const noexcept { return hasVal; } constexpr bool hasValue() const noexcept { return hasVal; } - T &getValue() LLVM_LVALUE_FUNCTION noexcept { + T &value() &noexcept { + assert(hasVal); + return val; + } + T &getValue() &noexcept { + assert(hasVal); + return val; + } + constexpr T const &value() const &noexcept { assert(hasVal); - return value; + return val; } - constexpr T const &getValue() const LLVM_LVALUE_FUNCTION noexcept { + constexpr T const &getValue() const &noexcept { assert(hasVal); - return value; + return val; } -#if LLVM_HAS_RVALUE_REFERENCE_THIS - T &&getValue() && noexcept { + T &&value() &&noexcept { assert(hasVal); - return std::move(value); + return std::move(val); + } + T &&getValue() &&noexcept { + assert(hasVal); + return std::move(val); } -#endif - template <class... Args> void emplace(Args &&... args) { + template <class... Args> void emplace(Args &&...args) { reset(); - ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...); + ::new ((void *)std::addressof(val)) T(std::forward<Args>(args)...); hasVal = true; } OptionalStorage &operator=(T const &y) { - if (hasValue()) { - value = y; + if (has_value()) { + val = y; } else { - ::new ((void *)std::addressof(value)) T(y); + ::new ((void *)std::addressof(val)) T(y); hasVal = true; } return *this; } OptionalStorage &operator=(T &&y) { - if (hasValue()) { - value = std::move(y); + if (has_value()) { + val = std::move(y); } else { - ::new ((void *)std::addressof(value)) T(std::move(y)); + ::new ((void *)std::addressof(val)) T(std::move(y)); hasVal = true; } return *this; } OptionalStorage &operator=(OptionalStorage const &other) { - if (other.hasValue()) { - if (hasValue()) { - value = other.value; + if (other.has_value()) { + if (has_value()) { + val = other.val; } else { - ::new ((void *)std::addressof(value)) T(other.value); + ::new ((void *)std::addressof(val)) T(other.val); hasVal = true; } } else { @@ -148,11 +159,11 @@ public: } OptionalStorage &operator=(OptionalStorage &&other) { - if (other.hasValue()) { - if (hasValue()) { - value = std::move(other.value); + if (other.has_value()) { + if (has_value()) { + val = std::move(other.val); } else { - ::new ((void *)std::addressof(value)) T(std::move(other.value)); + ::new ((void *)std::addressof(val)) T(std::move(other.val)); hasVal = true; } } else { @@ -165,7 +176,7 @@ public: template <typename T> class OptionalStorage<T, true> { union { char empty; - T value; + T val; }; bool hasVal = false; @@ -181,53 +192,64 @@ public: OptionalStorage &operator=(OptionalStorage &&other) = default; template <class... Args> - constexpr explicit OptionalStorage(in_place_t, Args &&... args) - : value(std::forward<Args>(args)...), hasVal(true) {} + constexpr explicit OptionalStorage(in_place_t, Args &&...args) + : val(std::forward<Args>(args)...), hasVal(true) {} void reset() noexcept { if (hasVal) { - value.~T(); + val.~T(); hasVal = false; } } + constexpr bool has_value() const noexcept { return hasVal; } constexpr bool hasValue() const noexcept { return hasVal; } - T &getValue() LLVM_LVALUE_FUNCTION noexcept { + T &value() &noexcept { + assert(hasVal); + return val; + } + T &getValue() &noexcept { assert(hasVal); - return value; + return val; } - constexpr T const &getValue() const LLVM_LVALUE_FUNCTION noexcept { + constexpr T const &value() const &noexcept { assert(hasVal); - return value; + return val; } -#if LLVM_HAS_RVALUE_REFERENCE_THIS - T &&getValue() && noexcept { + constexpr T const &getValue() const &noexcept { assert(hasVal); - return std::move(value); + return val; + } + T &&value() &&noexcept { + assert(hasVal); + return std::move(val); + } + T &&getValue() &&noexcept { + assert(hasVal); + return std::move(val); } -#endif - template <class... Args> void emplace(Args &&... args) { + template <class... Args> void emplace(Args &&...args) { reset(); - ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...); + ::new ((void *)std::addressof(val)) T(std::forward<Args>(args)...); hasVal = true; } OptionalStorage &operator=(T const &y) { - if (hasValue()) { - value = y; + if (has_value()) { + val = y; } else { - ::new ((void *)std::addressof(value)) T(y); + ::new ((void *)std::addressof(val)) T(y); hasVal = true; } return *this; } OptionalStorage &operator=(T &&y) { - if (hasValue()) { - value = std::move(y); + if (has_value()) { + val = std::move(y); } else { - ::new ((void *)std::addressof(value)) T(std::move(y)); + ::new ((void *)std::addressof(val)) T(std::move(y)); hasVal = true; } return *this; @@ -278,52 +300,55 @@ public: void reset() { Storage.reset(); } - constexpr const T *getPointer() const { return &Storage.getValue(); } - T *getPointer() { return &Storage.getValue(); } - constexpr const T &getValue() const LLVM_LVALUE_FUNCTION { - return Storage.getValue(); - } - T &getValue() LLVM_LVALUE_FUNCTION { return Storage.getValue(); } + constexpr const T *getPointer() const { return &Storage.value(); } + T *getPointer() { return &Storage.value(); } + constexpr const T &value() const & { return Storage.value(); } + constexpr const T &getValue() const & { return Storage.value(); } + T &value() & { return Storage.value(); } + T &getValue() & { return Storage.value(); } - constexpr explicit operator bool() const { return hasValue(); } - constexpr bool hasValue() const { return Storage.hasValue(); } + constexpr explicit operator bool() const { return has_value(); } + constexpr bool has_value() const { return Storage.has_value(); } + constexpr bool hasValue() const { return Storage.has_value(); } constexpr const T *operator->() const { return getPointer(); } T *operator->() { return getPointer(); } - constexpr const T &operator*() const LLVM_LVALUE_FUNCTION { - return getValue(); - } - T &operator*() LLVM_LVALUE_FUNCTION { return getValue(); } + constexpr const T &operator*() const & { return value(); } + T &operator*() & { return value(); } - template <typename U> - constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION { - return hasValue() ? getValue() : std::forward<U>(value); + template <typename U> constexpr T value_or(U &&alt) const & { + return has_value() ? value() : std::forward<U>(alt); + } + template <typename U> constexpr T getValueOr(U &&alt) const & { + return has_value() ? value() : std::forward<U>(alt); } /// Apply a function to the value if present; otherwise return None. template <class Function> - auto map(const Function &F) const LLVM_LVALUE_FUNCTION - -> Optional<decltype(F(getValue()))> { - if (*this) return F(getValue()); + auto map(const Function &F) const & -> Optional<decltype(F(value()))> { + if (*this) + return F(value()); return None; } -#if LLVM_HAS_RVALUE_REFERENCE_THIS - T &&getValue() && { return std::move(Storage.getValue()); } - T &&operator*() && { return std::move(Storage.getValue()); } + T &&value() && { return std::move(Storage.value()); } + T &&getValue() && { return std::move(Storage.value()); } + T &&operator*() && { return std::move(Storage.value()); } - template <typename U> - T getValueOr(U &&value) && { - return hasValue() ? std::move(getValue()) : std::forward<U>(value); + template <typename U> T value_or(U &&alt) && { + return has_value() ? std::move(value()) : std::forward<U>(alt); + } + template <typename U> T getValueOr(U &&alt) && { + return has_value() ? std::move(value()) : std::forward<U>(alt); } /// Apply a function to the value if present; otherwise return None. template <class Function> - auto map(const Function &F) && - -> Optional<decltype(F(std::move(*this).getValue()))> { - if (*this) return F(std::move(*this).getValue()); + auto map(const Function &F) + && -> Optional<decltype(F(std::move(*this).value()))> { + if (*this) + return F(std::move(*this).value()); return None; } -#endif }; template <class T> llvm::hash_code hash_value(const Optional<T> &O) { @@ -334,7 +359,7 @@ template <typename T, typename U> constexpr bool operator==(const Optional<T> &X, const Optional<U> &Y) { if (X && Y) return *X == *Y; - return X.hasValue() == Y.hasValue(); + return X.has_value() == Y.has_value(); } template <typename T, typename U> @@ -346,7 +371,7 @@ template <typename T, typename U> constexpr bool operator<(const Optional<T> &X, const Optional<U> &Y) { if (X && Y) return *X < *Y; - return X.hasValue() < Y.hasValue(); + return X.has_value() < Y.has_value(); } template <typename T, typename U> @@ -389,7 +414,7 @@ template <typename T> constexpr bool operator<(const Optional<T> &, NoneType) { } template <typename T> constexpr bool operator<(NoneType, const Optional<T> &X) { - return X.hasValue(); + return X.has_value(); } template <typename T> diff --git a/llvm/include/llvm/ADT/PointerIntPair.h b/llvm/include/llvm/ADT/PointerIntPair.h index b7ddf8855605..7d10b2a6dd14 100644 --- a/llvm/include/llvm/ADT/PointerIntPair.h +++ b/llvm/include/llvm/ADT/PointerIntPair.h @@ -61,19 +61,19 @@ public: IntType getInt() const { return (IntType)Info::getInt(Value); } - void setPointer(PointerTy PtrVal) LLVM_LVALUE_FUNCTION { + void setPointer(PointerTy PtrVal) & { Value = Info::updatePointer(Value, PtrVal); } - void setInt(IntType IntVal) LLVM_LVALUE_FUNCTION { + void setInt(IntType IntVal) & { Value = Info::updateInt(Value, static_cast<intptr_t>(IntVal)); } - void initWithPointer(PointerTy PtrVal) LLVM_LVALUE_FUNCTION { + void initWithPointer(PointerTy PtrVal) & { Value = Info::updatePointer(0, PtrVal); } - void setPointerAndInt(PointerTy PtrVal, IntType IntVal) LLVM_LVALUE_FUNCTION { + void setPointerAndInt(PointerTy PtrVal, IntType IntVal) & { Value = Info::updateInt(Info::updatePointer(0, PtrVal), static_cast<intptr_t>(IntVal)); } @@ -91,7 +91,7 @@ public: void *getOpaqueValue() const { return reinterpret_cast<void *>(Value); } - void setFromOpaqueValue(void *Val) LLVM_LVALUE_FUNCTION { + void setFromOpaqueValue(void *Val) & { Value = reinterpret_cast<intptr_t>(Val); } diff --git a/llvm/include/llvm/ADT/PointerSumType.h b/llvm/include/llvm/ADT/PointerSumType.h index a7ef774e205e..57f045035a78 100644 --- a/llvm/include/llvm/ADT/PointerSumType.h +++ b/llvm/include/llvm/ADT/PointerSumType.h @@ -272,11 +272,12 @@ struct DenseMapInfo<PointerSumType<TagT, MemberTs...>> { using SomePointerInfo = DenseMapInfo<SomePointerT>; static inline SumType getEmptyKey() { - return SumType::create<SomeTag>(SomePointerInfo::getEmptyKey()); + return SumType::template create<SomeTag>(SomePointerInfo::getEmptyKey()); } static inline SumType getTombstoneKey() { - return SumType::create<SomeTag>(SomePointerInfo::getTombstoneKey()); + return SumType::template create<SomeTag>( + SomePointerInfo::getTombstoneKey()); } static unsigned getHashValue(const SumType &Arg) { diff --git a/llvm/include/llvm/ADT/PointerUnion.h b/llvm/include/llvm/ADT/PointerUnion.h index 04d566bbc75e..f01db09dd765 100644 --- a/llvm/include/llvm/ADT/PointerUnion.h +++ b/llvm/include/llvm/ADT/PointerUnion.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include <algorithm> #include <cassert> @@ -87,6 +88,9 @@ namespace pointer_union_detail { }; } +// This is a forward declaration of CastInfoPointerUnionImpl +// Refer to its definition below for further details +template <typename... PTs> struct CastInfoPointerUnionImpl; /// A discriminated union of two or more pointer types, with the discriminator /// in the low bit of the pointer. /// @@ -122,6 +126,11 @@ class PointerUnion using First = TypeAtIndex<0, PTs...>; using Base = typename PointerUnion::PointerUnionMembers; + /// This is needed to give the CastInfo implementation below access + /// to protected members. + /// Refer to its definition for further details. + friend struct CastInfoPointerUnionImpl<PTs...>; + public: PointerUnion() = default; @@ -134,25 +143,24 @@ public: explicit operator bool() const { return !isNull(); } + // FIXME: Replace the uses of is(), get() and dyn_cast() with + // isa<T>, cast<T> and the llvm::dyn_cast<T> + /// Test if the Union currently holds the type matching T. - template <typename T> bool is() const { - return this->Val.getInt() == FirstIndexOfType<T, PTs...>::value; - } + template <typename T> inline bool is() const { return isa<T>(*this); } /// Returns the value of the specified pointer type. /// /// If the specified pointer type is incorrect, assert. - template <typename T> T get() const { - assert(is<T>() && "Invalid accessor called"); - return PointerLikeTypeTraits<T>::getFromVoidPointer(this->Val.getPointer()); + template <typename T> inline T get() const { + assert(isa<T>(*this) && "Invalid accessor called"); + return cast<T>(*this); } /// Returns the current pointer if it is of the specified pointer type, /// otherwise returns null. - template <typename T> T dyn_cast() const { - if (is<T>()) - return get<T>(); - return T(); + template <typename T> inline T dyn_cast() const { + return llvm::dyn_cast<T>(*this); } /// If the union is set to the first pointer type get an address pointing to @@ -205,6 +213,52 @@ bool operator<(PointerUnion<PTs...> lhs, PointerUnion<PTs...> rhs) { return lhs.getOpaqueValue() < rhs.getOpaqueValue(); } +/// We can't (at least, at this moment with C++14) declare CastInfo +/// as a friend of PointerUnion like this: +/// ``` +/// template<typename To> +/// friend struct CastInfo<To, PointerUnion<PTs...>>; +/// ``` +/// The compiler complains 'Partial specialization cannot be declared as a +/// friend'. +/// So we define this struct to be a bridge between CastInfo and +/// PointerUnion. +template <typename... PTs> struct CastInfoPointerUnionImpl { + using From = PointerUnion<PTs...>; + + template <typename To> static inline bool isPossible(From &F) { + return F.Val.getInt() == FirstIndexOfType<To, PTs...>::value; + } + + template <typename To> static To doCast(From &F) { + assert(isPossible<To>(F) && "cast to an incompatible type !"); + return PointerLikeTypeTraits<To>::getFromVoidPointer(F.Val.getPointer()); + } +}; + +// Specialization of CastInfo for PointerUnion +template <typename To, typename... PTs> +struct CastInfo<To, PointerUnion<PTs...>> + : public DefaultDoCastIfPossible<To, PointerUnion<PTs...>, + CastInfo<To, PointerUnion<PTs...>>> { + using From = PointerUnion<PTs...>; + using Impl = CastInfoPointerUnionImpl<PTs...>; + + static inline bool isPossible(From &f) { + return Impl::template isPossible<To>(f); + } + + static To doCast(From &f) { return Impl::template doCast<To>(f); } + + static inline To castFailed() { return To(); } +}; + +template <typename To, typename... PTs> +struct CastInfo<To, const PointerUnion<PTs...>> + : public ConstStrippingForwardingCast<To, const PointerUnion<PTs...>, + CastInfo<To, PointerUnion<PTs...>>> { +}; + // Teach SmallPtrSet that PointerUnion is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits)-1. template <typename ...PTs> diff --git a/llvm/include/llvm/ADT/SCCIterator.h b/llvm/include/llvm/ADT/SCCIterator.h index ad35e09f0f74..e4035a02b5f5 100644 --- a/llvm/include/llvm/ADT/SCCIterator.h +++ b/llvm/include/llvm/ADT/SCCIterator.h @@ -348,9 +348,14 @@ scc_member_iterator<GraphT, GT>::scc_member_iterator( NodeInfoMap[Edge->Target].Visited = false; std::queue<NodeType *> Queue; - for (auto &Node : NodeInfoMap) - if (Node.second.Visited) - Queue.push(Node.first); + // Initialze the queue with MST roots. Note that walking through SortedEdges + // instead of NodeInfoMap ensures an ordered deterministic push. + for (auto *Edge : SortedEdges) { + if (NodeInfoMap[Edge->Source].Visited) { + Queue.push(Edge->Source); + NodeInfoMap[Edge->Source].Visited = false; + } + } while (!Queue.empty()) { auto *Node = Queue.front(); diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index e2972f4f902a..0efa96e69a8c 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -129,7 +129,7 @@ struct function_traits<ReturnType (ClassType::*)(Args...) const, false> { /// Overload for class function types. template <typename ClassType, typename ReturnType, typename... Args> struct function_traits<ReturnType (ClassType::*)(Args...), false> - : function_traits<ReturnType (ClassType::*)(Args...) const> {}; + : public function_traits<ReturnType (ClassType::*)(Args...) const> {}; /// Overload for non-class function types. template <typename ReturnType, typename... Args> struct function_traits<ReturnType (*)(Args...), false> { @@ -143,6 +143,9 @@ struct function_traits<ReturnType (*)(Args...), false> { template <size_t i> using arg_t = typename std::tuple_element<i, std::tuple<Args...>>::type; }; +template <typename ReturnType, typename... Args> +struct function_traits<ReturnType (*const)(Args...), false> + : public function_traits<ReturnType (*)(Args...)> {}; /// Overload for non-class function type references. template <typename ReturnType, typename... Args> struct function_traits<ReturnType (&)(Args...), false> @@ -203,6 +206,17 @@ struct FirstIndexOfType<T, T, Us...> : std::integral_constant<size_t, 0> {}; template <size_t I, typename... Ts> using TypeAtIndex = std::tuple_element_t<I, std::tuple<Ts...>>; +/// Helper which adds two underlying types of enumeration type. +/// Implicit conversion to a common type is accepted. +template <typename EnumTy1, typename EnumTy2, + typename UT1 = std::enable_if_t<std::is_enum<EnumTy1>::value, + std::underlying_type_t<EnumTy1>>, + typename UT2 = std::enable_if_t<std::is_enum<EnumTy2>::value, + std::underlying_type_t<EnumTy2>>> +constexpr auto addEnumValues(EnumTy1 LHS, EnumTy2 RHS) { + return static_cast<UT1>(LHS) + static_cast<UT2>(RHS); +} + //===----------------------------------------------------------------------===// // Extra additions to <iterator> //===----------------------------------------------------------------------===// @@ -268,6 +282,13 @@ template <typename T> auto drop_begin(T &&RangeOrContainer, size_t N = 1) { adl_end(RangeOrContainer)); } +/// Return a range covering \p RangeOrContainer with the last N elements +/// excluded. +template <typename T> auto drop_end(T &&RangeOrContainer, size_t N = 1) { + return make_range(adl_begin(RangeOrContainer), + std::prev(adl_end(RangeOrContainer), N)); +} + // mapped_iterator - This is a simple iterator adapter that causes a function to // be applied whenever operator* is invoked on the iterator. @@ -423,6 +444,16 @@ public: findNextValid(); return *this; } + + decltype(auto) operator*() const { + assert(BaseT::wrapped() != End && "Cannot dereference end iterator!"); + return BaseT::operator*(); + } + + decltype(auto) operator->() const { + assert(BaseT::wrapped() != End && "Cannot dereference end iterator!"); + return BaseT::operator->(); + } }; /// Specialization of filter_iterator_base for forward iteration only. @@ -1160,13 +1191,15 @@ public: } /// Compare this range with another. - template <typename OtherT> bool operator==(const OtherT &other) const { - return size() == - static_cast<size_t>(std::distance(other.begin(), other.end())) && - std::equal(begin(), end(), other.begin()); + template <typename OtherT> + friend bool operator==(const indexed_accessor_range_base &lhs, + const OtherT &rhs) { + return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); } - template <typename OtherT> bool operator!=(const OtherT &other) const { - return !(*this == other); + template <typename OtherT> + friend bool operator!=(const indexed_accessor_range_base &lhs, + const OtherT &rhs) { + return !(lhs == rhs); } /// Return the size of this range. @@ -1650,6 +1683,15 @@ bool is_contained(R &&Range, const E &Element) { return std::find(adl_begin(Range), adl_end(Range), Element) != adl_end(Range); } +template <typename T> +constexpr bool is_contained(std::initializer_list<T> Set, T Value) { + // TODO: Use std::find when we switch to C++20. + for (T V : Set) + if (V == Value) + return true; + return false; +} + /// Wrapper function around std::is_sorted to check if elements in a range \p R /// are sorted with respect to a comparator \p C. template <typename R, typename Compare> bool is_sorted(R &&Range, Compare C) { diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index a4a790323a6b..e34702bdbb3c 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// /// -/// /file +/// \file /// This file defines the SmallVector class. /// //===----------------------------------------------------------------------===// @@ -949,6 +949,9 @@ public: return std::lexicographical_compare(this->begin(), this->end(), RHS.begin(), RHS.end()); } + bool operator>(const SmallVectorImpl &RHS) const { return RHS < *this; } + bool operator<=(const SmallVectorImpl &RHS) const { return !(*this > RHS); } + bool operator>=(const SmallVectorImpl &RHS) const { return !(*this < RHS); } }; template <typename T> diff --git a/llvm/include/llvm/ADT/Statistic.h b/llvm/include/llvm/ADT/Statistic.h index c39e161bcbcd..6c195cc44990 100644 --- a/llvm/include/llvm/ADT/Statistic.h +++ b/llvm/include/llvm/ADT/Statistic.h @@ -53,7 +53,7 @@ public: const char *const Name; const char *const Desc; - std::atomic<unsigned> Value; + std::atomic<uint64_t> Value; std::atomic<bool> Initialized; constexpr TrackingStatistic(const char *DebugType, const char *Name, @@ -65,12 +65,12 @@ public: const char *getName() const { return Name; } const char *getDesc() const { return Desc; } - unsigned getValue() const { return Value.load(std::memory_order_relaxed); } + uint64_t getValue() const { return Value.load(std::memory_order_relaxed); } // Allow use of this class as the value itself. - operator unsigned() const { return getValue(); } + operator uint64_t() const { return getValue(); } - const TrackingStatistic &operator=(unsigned Val) { + const TrackingStatistic &operator=(uint64_t Val) { Value.store(Val, std::memory_order_relaxed); return init(); } @@ -80,7 +80,7 @@ public: return init(); } - unsigned operator++(int) { + uint64_t operator++(int) { init(); return Value.fetch_add(1, std::memory_order_relaxed); } @@ -90,27 +90,27 @@ public: return init(); } - unsigned operator--(int) { + uint64_t operator--(int) { init(); return Value.fetch_sub(1, std::memory_order_relaxed); } - const TrackingStatistic &operator+=(unsigned V) { + const TrackingStatistic &operator+=(uint64_t V) { if (V == 0) return *this; Value.fetch_add(V, std::memory_order_relaxed); return init(); } - const TrackingStatistic &operator-=(unsigned V) { + const TrackingStatistic &operator-=(uint64_t V) { if (V == 0) return *this; Value.fetch_sub(V, std::memory_order_relaxed); return init(); } - void updateMax(unsigned V) { - unsigned PrevMax = Value.load(std::memory_order_relaxed); + void updateMax(uint64_t V) { + uint64_t PrevMax = Value.load(std::memory_order_relaxed); // Keep trying to update max until we succeed or another thread produces // a bigger max than us. while (V > PrevMax && !Value.compare_exchange_weak( @@ -134,26 +134,26 @@ public: NoopStatistic(const char * /*DebugType*/, const char * /*Name*/, const char * /*Desc*/) {} - unsigned getValue() const { return 0; } + uint64_t getValue() const { return 0; } // Allow use of this class as the value itself. - operator unsigned() const { return 0; } + operator uint64_t() const { return 0; } - const NoopStatistic &operator=(unsigned Val) { return *this; } + const NoopStatistic &operator=(uint64_t Val) { return *this; } const NoopStatistic &operator++() { return *this; } - unsigned operator++(int) { return 0; } + uint64_t operator++(int) { return 0; } const NoopStatistic &operator--() { return *this; } - unsigned operator--(int) { return 0; } + uint64_t operator--(int) { return 0; } - const NoopStatistic &operator+=(const unsigned &V) { return *this; } + const NoopStatistic &operator+=(const uint64_t &V) { return *this; } - const NoopStatistic &operator-=(const unsigned &V) { return *this; } + const NoopStatistic &operator-=(const uint64_t &V) { return *this; } - void updateMax(unsigned V) {} + void updateMax(uint64_t V) {} }; #if LLVM_ENABLE_STATS @@ -200,7 +200,7 @@ void PrintStatisticsJSON(raw_ostream &OS); /// during it's execution. It will return the value at the point that it is /// read. However, it will prevent new statistics from registering until it /// completes. -const std::vector<std::pair<StringRef, unsigned>> GetStatistics(); +const std::vector<std::pair<StringRef, uint64_t>> GetStatistics(); /// Reset the statistics. This can be used to zero and de-register the /// statistics in order to measure a compilation. diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 118def2f43e1..80ba47dd619c 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -240,6 +240,10 @@ namespace llvm { unsigned edit_distance(StringRef Other, bool AllowReplacements = true, unsigned MaxEditDistance = 0) const; + LLVM_NODISCARD unsigned + edit_distance_insensitive(StringRef Other, bool AllowReplacements = true, + unsigned MaxEditDistance = 0) const; + /// str - Get the contents as an std::string. LLVM_NODISCARD std::string str() const { diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h index 42277c013035..9d85a28fbf04 100644 --- a/llvm/include/llvm/ADT/Triple.h +++ b/llvm/include/llvm/ADT/Triple.h @@ -56,7 +56,10 @@ public: bpfel, // eBPF or extended BPF or 64-bit BPF (little endian) bpfeb, // eBPF or extended BPF or 64-bit BPF (big endian) csky, // CSKY: csky + dxil, // DXIL 32-bit DirectX bytecode hexagon, // Hexagon: hexagon + loongarch32, // LoongArch (32-bit): loongarch32 + loongarch64, // LoongArch (64-bit): loongarch64 m68k, // M68k: Motorola 680x0 family mips, // MIPS: mips, mipsallegrex, mipsr6 mipsel, // MIPSEL: mipsel, mipsallegrexe, mipsr6el @@ -146,7 +149,15 @@ public: MipsSubArch_r6, - PPCSubArch_spe + PPCSubArch_spe, + + // SPIR-V sub-arch corresponds to its version. + SPIRVSubArch_v10, + SPIRVSubArch_v11, + SPIRVSubArch_v12, + SPIRVSubArch_v13, + SPIRVSubArch_v14, + SPIRVSubArch_v15, }; enum VendorType { UnknownVendor, @@ -195,9 +206,11 @@ public: NVCL, // NVIDIA OpenCL AMDHSA, // AMD HSA Runtime PS4, + PS5, ELFIAMCU, TvOS, // Apple tvOS WatchOS, // Apple watchOS + DriverKit, // Apple DriverKit Mesa3D, Contiki, AMDPAL, // AMD PAL Runtime @@ -205,7 +218,8 @@ public: Hurd, // GNU/Hurd WASI, // Experimental WebAssembly OS Emscripten, - LastOSType = Emscripten + ShaderModel, // DirectX ShaderModel + LastOSType = ShaderModel }; enum EnvironmentType { UnknownEnvironment, @@ -232,15 +246,35 @@ public: CoreCLR, Simulator, // Simulator variants of other systems, e.g., Apple's iOS MacABI, // Mac Catalyst variant of Apple's iOS deployment target. - LastEnvironmentType = MacABI + + // Shader Stages + Pixel, + Vertex, + Geometry, + Hull, + Domain, + Compute, + Library, + RayGeneration, + Intersection, + AnyHit, + ClosestHit, + Miss, + Callable, + Mesh, + Amplification, + + LastEnvironmentType = Amplification }; enum ObjectFormatType { UnknownObjectFormat, COFF, + DXContainer, ELF, GOFF, MachO, + SPIRV, Wasm, XCOFF, }; @@ -360,6 +394,9 @@ public: /// with WatchOS or generic triples. VersionTuple getWatchOSVersion() const; + /// Parse the version number as with getOSVersion. + VersionTuple getDriverKitVersion() const; + /// @} /// @name Direct Component Access /// @{ @@ -462,11 +499,14 @@ public: return getSubArch() == Triple::ARMSubArch_v7k; } + /// Is this an Apple DriverKit triple. + bool isDriverKit() const { return getOS() == Triple::DriverKit; } + bool isOSzOS() const { return getOS() == Triple::ZOS; } - /// Is this a "Darwin" OS (macOS, iOS, tvOS or watchOS). + /// Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, or DriverKit). bool isOSDarwin() const { - return isMacOSX() || isiOS() || isWatchOS(); + return isMacOSX() || isiOS() || isWatchOS() || isDriverKit(); } bool isSimulatorEnvironment() const { @@ -640,19 +680,23 @@ public: return getObjectFormat() == Triple::XCOFF; } - /// Tests whether the target is the PS4 CPU - bool isPS4CPU() const { + /// Tests whether the target is the PS4 platform. + bool isPS4() const { return getArch() == Triple::x86_64 && getVendor() == Triple::SCEI && getOS() == Triple::PS4; } - /// Tests whether the target is the PS4 platform - bool isPS4() const { - return getVendor() == Triple::SCEI && - getOS() == Triple::PS4; + /// Tests whether the target is the PS5 platform. + bool isPS5() const { + return getArch() == Triple::x86_64 && + getVendor() == Triple::SCEI && + getOS() == Triple::PS5; } + /// Tests whether the target is the PS4 or PS5 platform. + bool isPS() const { return isPS4() || isPS5(); } + /// Tests whether the target is Android bool isAndroid() const { return getEnvironment() == Triple::Android; } @@ -676,6 +720,11 @@ public: getEnvironment() == Triple::MuslX32; } + /// Tests whether the target is DXIL. + bool isDXIL() const { + return getArch() == Triple::dxil; + } + /// Tests whether the target is SPIR (32- or 64-bit). bool isSPIR() const { return getArch() == Triple::spir || getArch() == Triple::spir64; @@ -774,6 +823,11 @@ public: : PointerWidth == 64; } + /// Tests whether the target is LoongArch (32- and 64-bit). + bool isLoongArch() const { + return getArch() == Triple::loongarch32 || getArch() == Triple::loongarch64; + } + /// Tests whether the target is MIPS 32-bit (little and big endian). bool isMIPS32() const { return getArch() == Triple::mips || getArch() == Triple::mipsel; @@ -810,6 +864,17 @@ public: return getArch() == Triple::riscv32 || getArch() == Triple::riscv64; } + /// Tests whether the target is 32-bit SPARC (little and big endian). + bool isSPARC32() const { + return getArch() == Triple::sparc || getArch() == Triple::sparcel; + } + + /// Tests whether the target is 64-bit SPARC (big endian). + bool isSPARC64() const { return getArch() == Triple::sparcv9; } + + /// Tests whether the target is SPARC. + bool isSPARC() const { return isSPARC32() || isSPARC64(); } + /// Tests whether the target is SystemZ. bool isSystemZ() const { return getArch() == Triple::systemz; @@ -863,7 +928,7 @@ public: } /// Tests if the environment supports dllimport/export annotations. - bool hasDLLImportExport() const { return isOSWindows() || isPS4CPU(); } + bool hasDLLImportExport() const { return isOSWindows() || isPS(); } /// @} /// @name Mutators @@ -971,7 +1036,7 @@ public: /// Get the "prefix" canonical name for the \p Kind architecture. This is the /// prefix used by the architecture specific builtins, and is suitable for - /// passing to \see Intrinsic::getIntrinsicForGCCBuiltin(). + /// passing to \see Intrinsic::getIntrinsicForClangBuiltin(). /// /// \return - The architecture prefix, or 0 if none is defined. static StringRef getArchTypePrefix(ArchType Kind); diff --git a/llvm/include/llvm/ADT/edit_distance.h b/llvm/include/llvm/ADT/edit_distance.h index c480c1e7cd78..6df3db6125d4 100644 --- a/llvm/include/llvm/ADT/edit_distance.h +++ b/llvm/include/llvm/ADT/edit_distance.h @@ -28,6 +28,9 @@ namespace llvm { /// /// \param ToArray the second sequence to compare. /// +/// \param Map A Functor to apply to each item of the sequences before +/// comparison. +/// /// \param AllowReplacements whether to allow element replacements (change one /// element into another) as a single operation, rather than as two operations /// (an insertion and a removal). @@ -39,10 +42,10 @@ namespace llvm { /// \returns the minimum number of element insertions, removals, or (if /// \p AllowReplacements is \c true) replacements needed to transform one of /// the given sequences into the other. If zero, the sequences are identical. -template<typename T> -unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, - bool AllowReplacements = true, - unsigned MaxEditDistance = 0) { +template <typename T, typename Functor> +unsigned ComputeMappedEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, + Functor Map, bool AllowReplacements = true, + unsigned MaxEditDistance = 0) { // The algorithm implemented below is the "classic" // dynamic-programming algorithm for computing the Levenshtein // distance, which is described here: @@ -58,6 +61,15 @@ unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, typename ArrayRef<T>::size_type m = FromArray.size(); typename ArrayRef<T>::size_type n = ToArray.size(); + if (MaxEditDistance) { + // If the difference in size between the 2 arrays is larger than the max + // distance allowed, we can bail out as we will always need at least + // MaxEditDistance insertions or removals. + typename ArrayRef<T>::size_type AbsDiff = m > n ? m - n : n - m; + if (AbsDiff > MaxEditDistance) + return MaxEditDistance + 1; + } + const unsigned SmallBufferSize = 64; unsigned SmallBuffer[SmallBufferSize]; std::unique_ptr<unsigned[]> Allocated; @@ -75,15 +87,16 @@ unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, unsigned BestThisRow = Row[0]; unsigned Previous = y - 1; + const auto &CurItem = Map(FromArray[y - 1]); for (typename ArrayRef<T>::size_type x = 1; x <= n; ++x) { int OldRow = Row[x]; if (AllowReplacements) { - Row[x] = std::min( - Previous + (FromArray[y-1] == ToArray[x-1] ? 0u : 1u), - std::min(Row[x-1], Row[x])+1); + Row[x] = std::min(Previous + (CurItem == Map(ToArray[x - 1]) ? 0u : 1u), + std::min(Row[x - 1], Row[x]) + 1); } else { - if (FromArray[y-1] == ToArray[x-1]) Row[x] = Previous; + if (CurItem == Map(ToArray[x - 1])) + Row[x] = Previous; else Row[x] = std::min(Row[x-1], Row[x]) + 1; } Previous = OldRow; @@ -98,6 +111,15 @@ unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, return Result; } +template <typename T> +unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray, + bool AllowReplacements = true, + unsigned MaxEditDistance = 0) { + return ComputeMappedEditDistance( + FromArray, ToArray, [](const T &X) -> const T & { return X; }, + AllowReplacements, MaxEditDistance); +} + } // End llvm namespace #endif diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index d4febe6c1db9..c065553db8e9 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -38,7 +38,6 @@ #define LLVM_ANALYSIS_ALIASANALYSIS_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/MemoryLocation.h" @@ -64,6 +63,7 @@ class LoopInfo; class PreservedAnalyses; class TargetLibraryInfo; class Value; +template <typename> class SmallPtrSetImpl; /// The possible results of an alias query. /// @@ -413,8 +413,12 @@ class EarliestEscapeInfo final : public CaptureInfo { /// This is used for cache invalidation purposes. DenseMap<Instruction *, TinyPtrVector<const Value *>> Inst2Obj; + const SmallPtrSetImpl<const Value *> &EphValues; + public: - EarliestEscapeInfo(DominatorTree &DT, const LoopInfo &LI) : DT(DT), LI(LI) {} + EarliestEscapeInfo(DominatorTree &DT, const LoopInfo &LI, + const SmallPtrSetImpl<const Value *> &EphValues) + : DT(DT), LI(LI), EphValues(EphValues) {} bool isNotCapturedBeforeOrAt(const Value *Object, const Instruction *I) override; @@ -1267,6 +1271,10 @@ bool isIdentifiedObject(const Value *V); /// IdentifiedObjects. bool isIdentifiedFunctionLocal(const Value *V); +/// Returns true if the pointer is one which would have been considered an +/// escape by isNonEscapingLocalObject. +bool isEscapeSource(const Value *V); + /// Return true if Object memory is not visible after an unwind, in the sense /// that program semantics cannot depend on Object containing any particular /// value on unwind. If the RequiresNoCaptureBeforeUnwind out parameter is set diff --git a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h index 2dd2e7ca916d..48181cc52626 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h +++ b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h @@ -24,12 +24,12 @@ #ifndef LLVM_ANALYSIS_ALIASANALYSISEVALUATOR_H #define LLVM_ANALYSIS_ALIASANALYSISEVALUATOR_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { class AAResults; +class Function; +class FunctionPass; class AAEvaluator : public PassInfoMixin<AAEvaluator> { int64_t FunctionCount = 0; diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h index b66ff395454d..78f5545ab215 100644 --- a/llvm/include/llvm/Analysis/AliasSetTracker.h +++ b/llvm/include/llvm/Analysis/AliasSetTracker.h @@ -22,13 +22,10 @@ #include "llvm/ADT/ilist_node.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/IR/Instruction.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Casting.h" #include <cassert> #include <cstddef> -#include <cstdint> #include <iterator> #include <vector> @@ -224,10 +221,6 @@ public: // track of the list's exact size. unsigned size() { return SetSize; } - /// If this alias set is known to contain a single instruction and *only* a - /// single unique instruction, return it. Otherwise, return nullptr. - Instruction* getUniqueInstruction(); - void print(raw_ostream &OS) const; void dump() const; diff --git a/llvm/include/llvm/Analysis/AssumeBundleQueries.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h index 77da19110246..785980130386 100644 --- a/llvm/include/llvm/Analysis/AssumeBundleQueries.h +++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h @@ -14,14 +14,14 @@ #ifndef LLVM_ANALYSIS_ASSUMEBUNDLEQUERIES_H #define LLVM_ANALYSIS_ASSUMEBUNDLEQUERIES_H -#include "llvm/IR/Attributes.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/IR/IntrinsicInst.h" namespace llvm { class AssumptionCache; class DominatorTree; +class Instruction; +class Value; /// Index of elements in the operand bundle. /// If the element exist it is guaranteed to be what is specified in this enum diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h index 97dda58109e9..46f14a21a9ff 100644 --- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h @@ -18,8 +18,6 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" -#include <algorithm> -#include <cstdint> #include <memory> #include <utility> diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 858dd369dd0b..d8e524d7cb80 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -20,6 +20,7 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseBitVector.h" #include "llvm/ADT/Twine.h" @@ -31,7 +32,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/ScaledNumber.h" #include "llvm/Support/raw_ostream.h" @@ -45,7 +45,6 @@ #include <list> #include <queue> #include <string> -#include <unordered_set> #include <utility> #include <vector> @@ -1300,7 +1299,7 @@ bool BlockFrequencyInfoImpl<BT>::computeMassInLoop(LoopData &Loop) { auto &HeaderNode = Loop.Nodes[H]; assert(!getBlock(HeaderNode)->getIrrLoopHeaderWeight() && "Shouldn't have a weight metadata"); - uint64_t MinWeight = MinHeaderWeight.getValue(); + uint64_t MinWeight = *MinHeaderWeight; LLVM_DEBUG(dbgs() << "Giving weight " << MinWeight << " to " << getBlockName(HeaderNode) << "\n"); if (MinWeight) @@ -1516,7 +1515,7 @@ void BlockFrequencyInfoImpl<BT>::findReachableBlocks( // Find all blocks to apply inference on, that is, reachable from the entry // along edges with non-zero probablities std::queue<const BlockT *> Queue; - std::unordered_set<const BlockT *> Reachable; + SmallPtrSet<const BlockT *, 8> Reachable; const BlockT *Entry = &F->front(); Queue.push(Entry); Reachable.insert(Entry); @@ -1527,16 +1526,14 @@ void BlockFrequencyInfoImpl<BT>::findReachableBlocks( auto EP = BPI->getEdgeProbability(SrcBB, DstBB); if (EP.isZero()) continue; - if (Reachable.find(DstBB) == Reachable.end()) { + if (Reachable.insert(DstBB).second) Queue.push(DstBB); - Reachable.insert(DstBB); - } } } // Find all blocks to apply inference on, that is, backward reachable from // the entry along (backward) edges with non-zero probablities - std::unordered_set<const BlockT *> InverseReachable; + SmallPtrSet<const BlockT *, 8> InverseReachable; for (const BlockT &BB : *F) { // An exit block is a block without any successors bool HasSucc = GraphTraits<const BlockT *>::child_begin(&BB) != @@ -1553,10 +1550,8 @@ void BlockFrequencyInfoImpl<BT>::findReachableBlocks( auto EP = BPI->getEdgeProbability(DstBB, SrcBB); if (EP.isZero()) continue; - if (InverseReachable.find(DstBB) == InverseReachable.end()) { + if (InverseReachable.insert(DstBB).second) Queue.push(DstBB); - InverseReachable.insert(DstBB); - } } } @@ -1581,15 +1576,14 @@ void BlockFrequencyInfoImpl<BT>::initTransitionProbabilities( // Find unique successors and corresponding probabilities for every block for (size_t Src = 0; Src < NumBlocks; Src++) { const BlockT *BB = Blocks[Src]; - std::unordered_set<const BlockT *> UniqueSuccs; + SmallPtrSet<const BlockT *, 2> UniqueSuccs; for (const auto SI : children<const BlockT *>(BB)) { // Ignore cold blocks if (BlockIndex.find(SI) == BlockIndex.end()) continue; // Ignore parallel edges between BB and SI blocks - if (UniqueSuccs.find(SI) != UniqueSuccs.end()) + if (!UniqueSuccs.insert(SI).second) continue; - UniqueSuccs.insert(SI); // Ignore jumps with zero probability auto EP = BPI->getEdgeProbability(BB, SI); if (EP.isZero()) @@ -1875,7 +1869,7 @@ struct BFIDOTGraphTraitsBase : public DefaultDOTGraphTraits { case GVDT_Count: { auto Count = Graph->getBlockProfileCount(Node); if (Count) - OS << Count.getValue(); + OS << *Count; else OS << "Unknown"; break; diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index e2099eba0f65..28418198acea 100644 --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -16,14 +16,12 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include "llvm/Support/BranchProbability.h" -#include "llvm/Support/Casting.h" #include <algorithm> #include <cassert> #include <cstdint> diff --git a/llvm/include/llvm/Analysis/CFGPrinter.h b/llvm/include/llvm/Analysis/CFGPrinter.h index c0cabceb4a54..768cda59c57d 100644 --- a/llvm/include/llvm/Analysis/CFGPrinter.h +++ b/llvm/include/llvm/Analysis/CFGPrinter.h @@ -18,7 +18,6 @@ #ifndef LLVM_ANALYSIS_CFGPRINTER_H #define LLVM_ANALYSIS_CFGPRINTER_H -#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/HeatUtils.h" @@ -27,10 +26,11 @@ #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/GraphWriter.h" namespace llvm { +template <class GraphType> struct GraphTraits; class CFGViewerPass : public PassInfoMixin<CFGViewerPass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); diff --git a/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h b/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h index 2eae2824bec3..6543c53c9b28 100644 --- a/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h +++ b/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h @@ -14,10 +14,12 @@ #ifndef LLVM_ANALYSIS_CFLALIASANALYSISUTILS_H #define LLVM_ANALYSIS_CFLALIASANALYSISUTILS_H +#include "llvm/IR/Argument.h" #include "llvm/IR/Function.h" #include "llvm/IR/ValueHandle.h" namespace llvm { + namespace cflaa { template <typename AAResult> struct FunctionHandle final : public CallbackVH { diff --git a/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h b/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h index 5f5e52af3d88..dfb363173187 100644 --- a/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h @@ -15,7 +15,6 @@ #define LLVM_ANALYSIS_CFLANDERSALIASANALYSIS_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/Optional.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFLAliasAnalysisUtils.h" #include "llvm/IR/PassManager.h" @@ -25,6 +24,7 @@ namespace llvm { +template <typename T> class Optional; class Function; class MemoryLocation; class TargetLibraryInfo; diff --git a/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h b/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h index ec05b3706ca3..865f4a54c094 100644 --- a/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h @@ -15,13 +15,11 @@ #define LLVM_ANALYSIS_CFLSTEENSALIASANALYSIS_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/Optional.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFLAliasAnalysisUtils.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" -#include "llvm/Support/Casting.h" #include <forward_list> #include <memory> diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h index 7cf172dc1dd1..9d1b331346b6 100644 --- a/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -88,27 +88,21 @@ #ifndef LLVM_ANALYSIS_CGSCCPASSMANAGER_H #define LLVM_ANALYSIS_CGSCCPASSMANAGER_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" -#include "llvm/ADT/PriorityWorklist.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/LazyCallGraph.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/InstIterator.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <utility> namespace llvm { +class Function; +class Value; +template <typename T, unsigned int N> class SmallPriorityWorklist; struct CGSCCUpdateResult; + class Module; // Allow debug logging in this inline function. @@ -278,16 +272,6 @@ struct CGSCCUpdateResult { /// the list and removing entries from it. SmallPtrSetImpl<LazyCallGraph::SCC *> &InvalidatedSCCs; - /// If non-null, the updated current \c RefSCC being processed. - /// - /// This is set when a graph refinement takes place and the "current" point - /// in the graph moves "down" or earlier in the post-order walk. This will - /// often cause the "current" RefSCC to be a newly created RefSCC object and - /// the old one to be added to the above worklist. When that happens, this - /// pointer is non-null and can be used to continue processing the "top" of - /// the post-order walk. - LazyCallGraph::RefSCC *UpdatedRC; - /// If non-null, the updated current \c SCC being processed. /// /// This is set when a graph refinement takes place and the "current" point diff --git a/llvm/include/llvm/Analysis/CallGraph.h b/llvm/include/llvm/Analysis/CallGraph.h index 4da448c9900b..88d56785de67 100644 --- a/llvm/include/llvm/Analysis/CallGraph.h +++ b/llvm/include/llvm/Analysis/CallGraph.h @@ -45,9 +45,6 @@ #ifndef LLVM_ANALYSIS_CALLGRAPH_H #define LLVM_ANALYSIS_CALLGRAPH_H -#include "llvm/ADT/GraphTraits.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/PassManager.h" @@ -61,7 +58,9 @@ namespace llvm { +template <class GraphType> struct GraphTraits; class CallGraphNode; +class Function; class Module; class raw_ostream; diff --git a/llvm/include/llvm/Analysis/CallPrinter.h b/llvm/include/llvm/Analysis/CallPrinter.h index 8d4159f3ddc0..d325d0010371 100644 --- a/llvm/include/llvm/Analysis/CallPrinter.h +++ b/llvm/include/llvm/Analysis/CallPrinter.h @@ -14,10 +14,24 @@ #ifndef LLVM_ANALYSIS_CALLPRINTER_H #define LLVM_ANALYSIS_CALLPRINTER_H +#include "llvm/IR/PassManager.h" + namespace llvm { class ModulePass; +/// Pass for printing the call graph to a dot file +class CallGraphDOTPrinterPass : public PassInfoMixin<CallGraphDOTPrinterPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +/// Pass for viewing the call graph +class CallGraphViewerPass : public PassInfoMixin<CallGraphViewerPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + ModulePass *createCallGraphViewerPass(); ModulePass *createCallGraphDOTPrinterPass(); diff --git a/llvm/include/llvm/Analysis/CaptureTracking.h b/llvm/include/llvm/Analysis/CaptureTracking.h index 50d12db7a1c3..a2d9277745e4 100644 --- a/llvm/include/llvm/Analysis/CaptureTracking.h +++ b/llvm/include/llvm/Analysis/CaptureTracking.h @@ -14,6 +14,7 @@ #define LLVM_ANALYSIS_CAPTURETRACKING_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace llvm { @@ -24,6 +25,7 @@ namespace llvm { class DominatorTree; class LoopInfo; class Function; + template <typename T> class SmallPtrSetImpl; /// getDefaultMaxUsesToExploreForCaptureTracking - Return default value of /// the maximal number of uses to explore before giving up. It is used by @@ -41,7 +43,13 @@ namespace llvm { /// one value before giving up due too "too many uses". If MaxUsesToExplore /// is zero, a default value is assumed. bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, + bool StoreCaptures, unsigned MaxUsesToExplore = 0); + + /// Variant of the above function which accepts a set of Values that are + /// ephemeral and cannot cause pointers to escape. + bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, bool StoreCaptures, + const SmallPtrSetImpl<const Value *> &EphValues, unsigned MaxUsesToExplore = 0); /// PointerMayBeCapturedBefore - Return true if this pointer value may be @@ -72,10 +80,11 @@ namespace llvm { // nullptr is returned. Note that the caller of the function has to ensure // that the instruction the result value is compared against is not in a // cycle. - Instruction *FindEarliestCapture(const Value *V, Function &F, - bool ReturnCaptures, bool StoreCaptures, - const DominatorTree &DT, - unsigned MaxUsesToExplore = 0); + Instruction * + FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures, + bool StoreCaptures, const DominatorTree &DT, + const SmallPtrSetImpl<const Value *> &EphValues, + unsigned MaxUsesToExplore = 0); /// This callback is used in conjunction with PointerMayBeCaptured. In /// addition to the interface here, you'll need to provide your own getters @@ -105,6 +114,24 @@ namespace llvm { virtual bool isDereferenceableOrNull(Value *O, const DataLayout &DL); }; + /// Types of use capture kinds, see \p DetermineUseCaptureKind. + enum class UseCaptureKind { + NO_CAPTURE, + MAY_CAPTURE, + PASSTHROUGH, + }; + + /// Determine what kind of capture behaviour \p U may exhibit. + /// + /// A use can be no-capture, a use can potentially capture, or a use can be + /// passthrough such that the uses of the user or \p U should be inspected. + /// The \p IsDereferenceableOrNull callback is used to rule out capturing for + /// certain comparisons. + UseCaptureKind + DetermineUseCaptureKind(const Use &U, + llvm::function_ref<bool(Value *, const DataLayout &)> + IsDereferenceableOrNull); + /// PointerMayBeCaptured - Visit the value and the values derived from it and /// find values which appear to be capturing the pointer value. This feeds /// results into and is controlled by the CaptureTracker object. diff --git a/llvm/include/llvm/Analysis/CmpInstAnalysis.h b/llvm/include/llvm/Analysis/CmpInstAnalysis.h index 3d34cd12aea4..332eb9b66e9c 100644 --- a/llvm/include/llvm/Analysis/CmpInstAnalysis.h +++ b/llvm/include/llvm/Analysis/CmpInstAnalysis.h @@ -17,7 +17,7 @@ #include "llvm/IR/InstrTypes.h" namespace llvm { - class ICmpInst; + class Type; class Value; /// Encode a icmp predicate into a three bit mask. These bits are carefully @@ -43,7 +43,7 @@ namespace llvm { /// 110 6 A <= B /// 111 7 Always true /// - unsigned getICmpCode(const ICmpInst *ICI, bool InvertPred = false); + unsigned getICmpCode(CmpInst::Predicate Pred); /// This is the complement of getICmpCode. It turns a predicate code into /// either a constant true or false or the predicate for a new ICmp. @@ -58,6 +58,39 @@ namespace llvm { /// equality comparison (which is signless). bool predicatesFoldable(CmpInst::Predicate P1, CmpInst::Predicate P2); + /// Similar to getICmpCode but for FCmpInst. This encodes a fcmp predicate + /// into a four bit mask. + inline unsigned getFCmpCode(CmpInst::Predicate CC) { + assert(CmpInst::FCMP_FALSE <= CC && CC <= CmpInst::FCMP_TRUE && + "Unexpected FCmp predicate!"); + // Take advantage of the bit pattern of CmpInst::Predicate here. + // U L G E + static_assert(CmpInst::FCMP_FALSE == 0, ""); // 0 0 0 0 + static_assert(CmpInst::FCMP_OEQ == 1, ""); // 0 0 0 1 + static_assert(CmpInst::FCMP_OGT == 2, ""); // 0 0 1 0 + static_assert(CmpInst::FCMP_OGE == 3, ""); // 0 0 1 1 + static_assert(CmpInst::FCMP_OLT == 4, ""); // 0 1 0 0 + static_assert(CmpInst::FCMP_OLE == 5, ""); // 0 1 0 1 + static_assert(CmpInst::FCMP_ONE == 6, ""); // 0 1 1 0 + static_assert(CmpInst::FCMP_ORD == 7, ""); // 0 1 1 1 + static_assert(CmpInst::FCMP_UNO == 8, ""); // 1 0 0 0 + static_assert(CmpInst::FCMP_UEQ == 9, ""); // 1 0 0 1 + static_assert(CmpInst::FCMP_UGT == 10, ""); // 1 0 1 0 + static_assert(CmpInst::FCMP_UGE == 11, ""); // 1 0 1 1 + static_assert(CmpInst::FCMP_ULT == 12, ""); // 1 1 0 0 + static_assert(CmpInst::FCMP_ULE == 13, ""); // 1 1 0 1 + static_assert(CmpInst::FCMP_UNE == 14, ""); // 1 1 1 0 + static_assert(CmpInst::FCMP_TRUE == 15, ""); // 1 1 1 1 + return CC; + } + + /// This is the complement of getFCmpCode. It turns a predicate code into + /// either a constant true or false or the predicate for a new FCmp. + /// Non-NULL return value will be a true or false constant. + /// NULL return means a new ICmp is needed. The predicate is output in Pred. + Constant *getPredForFCmpCode(unsigned Code, Type *OpTy, + CmpInst::Predicate &Pred); + /// Decompose an icmp into the form ((X & Mask) pred 0) if possible. The /// returned predicate is either == or !=. Returns false if decomposition /// fails. diff --git a/llvm/include/llvm/Analysis/CodeMetrics.h b/llvm/include/llvm/Analysis/CodeMetrics.h index 615591aa83ad..a9431bca1125 100644 --- a/llvm/include/llvm/Analysis/CodeMetrics.h +++ b/llvm/include/llvm/Analysis/CodeMetrics.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_CODEMETRICS_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Support/InstructionCost.h" namespace llvm { class AssumptionCache; @@ -47,14 +48,14 @@ struct CodeMetrics { /// True if this function calls alloca (in the C sense). bool usesDynamicAlloca = false; - /// Number of instructions in the analyzed blocks. - unsigned NumInsts = false; + /// Code size cost of the analyzed blocks. + InstructionCost NumInsts = 0; /// Number of analyzed blocks. unsigned NumBlocks = false; /// Keeps track of basic block code size estimates. - DenseMap<const BasicBlock *, unsigned> NumBBInsts; + DenseMap<const BasicBlock *, InstructionCost> NumBBInsts; /// Keep track of the number of calls to 'big' functions. unsigned NumCalls = false; diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h index 37258c80e3a3..23ec7d6b70ec 100644 --- a/llvm/include/llvm/Analysis/ConstantFolding.h +++ b/llvm/include/llvm/Analysis/ConstantFolding.h @@ -19,16 +19,18 @@ #ifndef LLVM_ANALYSIS_CONSTANTFOLDING_H #define LLVM_ANALYSIS_CONSTANTFOLDING_H +#include <stdint.h> + namespace llvm { class APInt; template <typename T> class ArrayRef; class CallBase; class Constant; -class ConstantExpr; class DSOLocalEquivalent; class DataLayout; class Function; class GlobalValue; +class GlobalVariable; class Instruction; class TargetLibraryInfo; class Type; @@ -65,14 +67,13 @@ Constant *ConstantFoldInstOperands(Instruction *I, ArrayRef<Constant *> Ops, const DataLayout &DL, const TargetLibraryInfo *TLI = nullptr); -/// ConstantFoldCompareInstOperands - Attempt to constant fold a compare -/// instruction (icmp/fcmp) with the specified operands. If it fails, it -/// returns a constant expression of the specified operands. -/// -Constant * -ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, - Constant *RHS, const DataLayout &DL, - const TargetLibraryInfo *TLI = nullptr); +/// Attempt to constant fold a compare instruction (icmp/fcmp) with the +/// specified operands. If it fails, it returns a constant expression of the +/// specified operands. +/// Denormal inputs may be flushed based on the denormal handling mode. +Constant *ConstantFoldCompareInstOperands( + unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, + const TargetLibraryInfo *TLI = nullptr, const Instruction *I = nullptr); /// Attempt to constant fold a unary operation with the specified /// operand. If it fails, it returns a constant expression of the specified @@ -86,6 +87,21 @@ Constant *ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, Constant *ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL); +/// Attempt to constant fold a floating point binary operation with the +/// specified operands, applying the denormal handling mod to the operands. If +/// it fails, it returns a constant expression of the specified operands. +Constant *ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS, + Constant *RHS, const DataLayout &DL, + const Instruction *I); + +/// Attempt to flush float point constant according to denormal mode set in the +/// instruction's parent function attributes. If so, return a zero with the +/// correct sign, otherwise return the original constant. Inputs and outputs to +/// floating point instructions can have their mode set separately, so the +/// direction is also needed. +Constant *FlushFPConstant(Constant *Operand, const Instruction *I, + bool IsOutput); + /// Attempt to constant fold a select instruction with the specified /// operands. The constant result is returned if successful; if not, null is /// returned. @@ -173,6 +189,8 @@ Constant *ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy, /// Check whether the given call has no side-effects. /// Specifically checks for math routimes which sometimes set errno. bool isMathLibCallNoop(const CallBase *Call, const TargetLibraryInfo *TLI); + +Constant *ReadByteArrayFromGlobal(const GlobalVariable *GV, uint64_t Offset); } #endif diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h index d7800f578325..2c83658b81dc 100644 --- a/llvm/include/llvm/Analysis/ConstraintSystem.h +++ b/llvm/include/llvm/Analysis/ConstraintSystem.h @@ -11,7 +11,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include <string> @@ -37,7 +36,7 @@ class ConstraintSystem { bool mayHaveSolutionImpl(); public: - bool addVariableRow(const SmallVector<int64_t, 8> &R) { + bool addVariableRow(ArrayRef<int64_t> R) { assert(Constraints.empty() || R.size() == Constraints.back().size()); // If all variable coefficients are 0, the constraint does not provide any // usable information. @@ -49,11 +48,16 @@ public: GCD = APIntOps::GreatestCommonDivisor({32, (uint32_t)A}, {32, GCD}) .getZExtValue(); } - Constraints.push_back(R); + Constraints.emplace_back(R.begin(), R.end()); return true; } - bool addVariableRowFill(const SmallVector<int64_t, 8> &R) { + bool addVariableRowFill(ArrayRef<int64_t> R) { + // If all variable coefficients are 0, the constraint does not provide any + // usable information. + if (all_of(makeArrayRef(R).drop_front(1), [](int64_t C) { return C == 0; })) + return false; + for (auto &CR : Constraints) { while (CR.size() != R.size()) CR.push_back(0); @@ -75,7 +79,14 @@ public: bool isConditionImplied(SmallVector<int64_t, 8> R) const; + ArrayRef<int64_t> getLastConstraint() { return Constraints[0]; } void popLastConstraint() { Constraints.pop_back(); } + void popLastNVariables(unsigned N) { + for (auto &C : Constraints) { + for (unsigned i = 0; i < N; i++) + C.pop_back(); + } + } /// Returns the number of rows in the constraint system. unsigned size() const { return Constraints.size(); } diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h index c5107da2a017..7649e630b23d 100644 --- a/llvm/include/llvm/Analysis/DDG.h +++ b/llvm/include/llvm/Analysis/DDG.h @@ -18,9 +18,11 @@ #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DependenceGraphBuilder.h" #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/IR/Instructions.h" namespace llvm { +class Function; +class Loop; +class LoopInfo; class DDGNode; class DDGEdge; using DDGNodeBase = DGNode<DDGNode, DDGEdge>; diff --git a/llvm/include/llvm/Analysis/DDGPrinter.h b/llvm/include/llvm/Analysis/DDGPrinter.h index 4477b387fe50..d93c28280bac 100644 --- a/llvm/include/llvm/Analysis/DDGPrinter.h +++ b/llvm/include/llvm/Analysis/DDGPrinter.h @@ -16,10 +16,11 @@ #define LLVM_ANALYSIS_DDGPRINTER_H #include "llvm/Analysis/DDG.h" -#include "llvm/Pass.h" #include "llvm/Support/DOTGraphTraits.h" namespace llvm { +class LPMUpdater; +class Loop; //===--------------------------------------------------------------------===// // Implementation of DDG DOT Printer for a loop. diff --git a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h index d8021907b5b2..c35e189de6fc 100644 --- a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -14,23 +14,156 @@ #define LLVM_ANALYSIS_DOTGRAPHTRAITSPASS_H #include "llvm/Analysis/CFGPrinter.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/GraphWriter.h" namespace llvm { /// Default traits class for extracting a graph from an analysis pass. /// +/// This assumes that 'GraphT' is 'AnalysisT::Result *', and pass it through +template <typename Result, typename GraphT = Result *> +struct DefaultAnalysisGraphTraits { + static GraphT getGraph(Result R) { return &R; } +}; + +template <typename GraphT> +void viewGraphForFunction(Function &F, GraphT Graph, StringRef Name, + bool IsSimple) { + std::string GraphName = DOTGraphTraits<GraphT *>::getGraphName(&Graph); + + ViewGraph(Graph, Name, IsSimple, + GraphName + " for '" + F.getName() + "' function"); +} + +template <typename AnalysisT, bool IsSimple, + typename GraphT = typename AnalysisT::Result *, + typename AnalysisGraphTraitsT = + DefaultAnalysisGraphTraits<typename AnalysisT::Result &, GraphT>> +struct DOTGraphTraitsViewer + : PassInfoMixin<DOTGraphTraitsViewer<AnalysisT, IsSimple, GraphT, + AnalysisGraphTraitsT>> { + DOTGraphTraitsViewer(StringRef GraphName) : Name(GraphName) {} + + /// Return true if this function should be processed. + /// + /// An implementation of this class my override this function to indicate that + /// only certain functions should be viewed. + /// + /// @param Result The current analysis result for this function. + virtual bool processFunction(Function &F, + const typename AnalysisT::Result &Result) { + return true; + } + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) { + auto &Result = FAM.getResult<AnalysisT>(F); + if (!processFunction(F, Result)) + return PreservedAnalyses::all(); + + GraphT Graph = AnalysisGraphTraitsT::getGraph(Result); + viewGraphForFunction(F, Graph, Name, IsSimple); + + return PreservedAnalyses::all(); + }; + +protected: + /// Avoid compiler warning "has virtual functions but non-virtual destructor + /// [-Wnon-virtual-dtor]" in derived classes. + /// + /// DOTGraphTraitsViewer is also used as a mixin for avoiding repeated + /// implementation of viewer passes, ie there should be no + /// runtime-polymorphisms/downcasting involving this class and hence no + /// virtual destructor needed. Making this dtor protected stops accidental + /// invocation when the derived class destructor should have been called. + /// Those derived classes sould be marked final to avoid the warning. + ~DOTGraphTraitsViewer() {} + +private: + StringRef Name; +}; + +template <typename GraphT> +void printGraphForFunction(Function &F, GraphT Graph, StringRef Name, + bool IsSimple) { + std::string Filename = Name.str() + "." + F.getName().str() + ".dot"; + std::error_code EC; + + errs() << "Writing '" << Filename << "'..."; + + raw_fd_ostream File(Filename, EC, sys::fs::OF_TextWithCRLF); + std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); + + if (!EC) + WriteGraph(File, Graph, IsSimple, + GraphName + " for '" + F.getName() + "' function"); + else + errs() << " error opening file for writing!"; + errs() << "\n"; +} + +template <typename AnalysisT, bool IsSimple, + typename GraphT = typename AnalysisT::Result *, + typename AnalysisGraphTraitsT = + DefaultAnalysisGraphTraits<typename AnalysisT::Result &, GraphT>> +struct DOTGraphTraitsPrinter + : PassInfoMixin<DOTGraphTraitsPrinter<AnalysisT, IsSimple, GraphT, + AnalysisGraphTraitsT>> { + DOTGraphTraitsPrinter(StringRef GraphName) : Name(GraphName) {} + + /// Return true if this function should be processed. + /// + /// An implementation of this class my override this function to indicate that + /// only certain functions should be viewed. + /// + /// @param Analysis The current analysis result for this function. + virtual bool processFunction(Function &F, + const typename AnalysisT::Result &Result) { + return true; + } + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) { + auto &Result = FAM.getResult<AnalysisT>(F); + if (!processFunction(F, Result)) + return PreservedAnalyses::all(); + + GraphT Graph = AnalysisGraphTraitsT::getGraph(Result); + + printGraphForFunction(F, Graph, Name, IsSimple); + + return PreservedAnalyses::all(); + }; + +protected: + /// Avoid compiler warning "has virtual functions but non-virtual destructor + /// [-Wnon-virtual-dtor]" in derived classes. + /// + /// DOTGraphTraitsPrinter is also used as a mixin for avoiding repeated + /// implementation of printer passes, ie there should be no + /// runtime-polymorphisms/downcasting involving this class and hence no + /// virtual destructor needed. Making this dtor protected stops accidental + /// invocation when the derived class destructor should have been called. + /// Those derived classes sould be marked final to avoid the warning. + ~DOTGraphTraitsPrinter() {} + +private: + StringRef Name; +}; + +/// Default traits class for extracting a graph from an analysis pass. +/// /// This assumes that 'GraphT' is 'AnalysisT *' and so just passes it through. template <typename AnalysisT, typename GraphT = AnalysisT *> -struct DefaultAnalysisGraphTraits { +struct LegacyDefaultAnalysisGraphTraits { static GraphT getGraph(AnalysisT *A) { return A; } }; -template < - typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > -class DOTGraphTraitsViewer : public FunctionPass { +template <typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, + typename AnalysisGraphTraitsT = + LegacyDefaultAnalysisGraphTraits<AnalysisT, GraphT>> +class DOTGraphTraitsViewerWrapperPass : public FunctionPass { public: - DOTGraphTraitsViewer(StringRef GraphName, char &ID) + DOTGraphTraitsViewerWrapperPass(StringRef GraphName, char &ID) : FunctionPass(ID), Name(GraphName) {} /// Return true if this function should be processed. @@ -50,10 +183,7 @@ public: return false; GraphT Graph = AnalysisGraphTraitsT::getGraph(&Analysis); - std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); - std::string Title = GraphName + " for '" + F.getName().str() + "' function"; - - ViewGraph(Graph, Name, IsSimple, Title); + viewGraphForFunction(F, Graph, Name, IsSimple); return false; } @@ -67,12 +197,12 @@ private: std::string Name; }; -template < - typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > -class DOTGraphTraitsPrinter : public FunctionPass { +template <typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, + typename AnalysisGraphTraitsT = + LegacyDefaultAnalysisGraphTraits<AnalysisT, GraphT>> +class DOTGraphTraitsPrinterWrapperPass : public FunctionPass { public: - DOTGraphTraitsPrinter(StringRef GraphName, char &ID) + DOTGraphTraitsPrinterWrapperPass(StringRef GraphName, char &ID) : FunctionPass(ID), Name(GraphName) {} /// Return true if this function should be processed. @@ -92,20 +222,7 @@ public: return false; GraphT Graph = AnalysisGraphTraitsT::getGraph(&Analysis); - std::string Filename = Name + "." + F.getName().str() + ".dot"; - std::error_code EC; - - errs() << "Writing '" << Filename << "'..."; - - raw_fd_ostream File(Filename, EC, sys::fs::OF_TextWithCRLF); - std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); - std::string Title = GraphName + " for '" + F.getName().str() + "' function"; - - if (!EC) - WriteGraph(File, Graph, IsSimple, Title); - else - errs() << " error opening file for writing!"; - errs() << "\n"; + printGraphForFunction(F, Graph, Name, IsSimple); return false; } @@ -119,12 +236,12 @@ private: std::string Name; }; -template < - typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > -class DOTGraphTraitsModuleViewer : public ModulePass { +template <typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, + typename AnalysisGraphTraitsT = + LegacyDefaultAnalysisGraphTraits<AnalysisT, GraphT>> +class DOTGraphTraitsModuleViewerWrapperPass : public ModulePass { public: - DOTGraphTraitsModuleViewer(StringRef GraphName, char &ID) + DOTGraphTraitsModuleViewerWrapperPass(StringRef GraphName, char &ID) : ModulePass(ID), Name(GraphName) {} bool runOnModule(Module &M) override { @@ -145,12 +262,12 @@ private: std::string Name; }; -template < - typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > -class DOTGraphTraitsModulePrinter : public ModulePass { +template <typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, + typename AnalysisGraphTraitsT = + LegacyDefaultAnalysisGraphTraits<AnalysisT, GraphT>> +class DOTGraphTraitsModulePrinterWrapperPass : public ModulePass { public: - DOTGraphTraitsModulePrinter(StringRef GraphName, char &ID) + DOTGraphTraitsModulePrinterWrapperPass(StringRef GraphName, char &ID) : ModulePass(ID), Name(GraphName) {} bool runOnModule(Module &M) override { diff --git a/llvm/include/llvm/Analysis/Delinearization.h b/llvm/include/llvm/Analysis/Delinearization.h index 6e942530f253..95a36b8b79a4 100644 --- a/llvm/include/llvm/Analysis/Delinearization.h +++ b/llvm/include/llvm/Analysis/Delinearization.h @@ -16,11 +16,11 @@ #ifndef LLVM_ANALYSIS_DELINEARIZATION_H #define LLVM_ANALYSIS_DELINEARIZATION_H -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/raw_ostream.h" namespace llvm { +class raw_ostream; +template <typename T> class SmallVectorImpl; class GetElementPtrInst; class ScalarEvolution; class SCEV; @@ -125,6 +125,17 @@ bool getIndexExpressionsFromGEP(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &Subscripts, SmallVectorImpl<int> &Sizes); +/// Implementation of fixed size array delinearization. Try to delinearize +/// access function for a fixed size multi-dimensional array, by deriving +/// subscripts from GEP instructions. Returns true upon success and false +/// otherwise. \p Inst is the load/store instruction whose pointer operand is +/// the one we want to delinearize. \p AccessFn is its corresponding SCEV +/// expression w.r.t. the surrounding loop. +bool tryDelinearizeFixedSizeImpl(ScalarEvolution *SE, Instruction *Inst, + const SCEV *AccessFn, + SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<int> &Sizes); + struct DelinearizationPrinterPass : public PassInfoMixin<DelinearizationPrinterPass> { explicit DelinearizationPrinterPass(raw_ostream &OS); diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h index 638f4869d677..a34afe9fb38d 100644 --- a/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -927,9 +927,9 @@ namespace llvm { bool tryDelinearize(Instruction *Src, Instruction *Dst, SmallVectorImpl<Subscript> &Pair); - /// Tries to delinearize access function for a fixed size multi-dimensional - /// array, by deriving subscripts from GEP instructions. Returns true upon - /// success and false otherwise. + /// Tries to delinearize \p Src and \p Dst access functions for a fixed size + /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to + /// delinearize \p Src and \p Dst separately, bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst, const SCEV *SrcAccessFn, const SCEV *DstAccessFn, diff --git a/llvm/include/llvm/Analysis/DivergenceAnalysis.h b/llvm/include/llvm/Analysis/DivergenceAnalysis.h index c52b42ae8dc2..4c2a5399ea54 100644 --- a/llvm/include/llvm/Analysis/DivergenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DivergenceAnalysis.h @@ -17,16 +17,16 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/Analysis/SyncDependenceAnalysis.h" -#include "llvm/IR/Function.h" -#include "llvm/Pass.h" +#include "llvm/IR/PassManager.h" #include <vector> namespace llvm { -class Value; +class Function; class Instruction; class Loop; class raw_ostream; class TargetTransformInfo; +class Value; /// \brief Generic divergence analysis for reducible CFGs. /// @@ -41,7 +41,7 @@ public: /// \param RegionLoop if non-null the analysis is restricted to \p RegionLoop. /// Otherwise the whole function is analyzed. /// \param IsLCSSAForm whether the analysis may assume that the IR in the - /// region in in LCSSA form. + /// region in LCSSA form. DivergenceAnalysisImpl(const Function &F, const Loop *RegionLoop, const DominatorTree &DT, const LoopInfo &LI, SyncDependenceAnalysis &SDA, bool IsLCSSAForm); diff --git a/llvm/include/llvm/Analysis/DomPrinter.h b/llvm/include/llvm/Analysis/DomPrinter.h index e6df12d88072..83fe721346ab 100644 --- a/llvm/include/llvm/Analysis/DomPrinter.h +++ b/llvm/include/llvm/Analysis/DomPrinter.h @@ -14,30 +14,120 @@ #ifndef LLVM_ANALYSIS_DOMPRINTER_H #define LLVM_ANALYSIS_DOMPRINTER_H +#include "llvm/Analysis/DOTGraphTraitsPass.h" +#include "llvm/Analysis/PostDominators.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/PassManager.h" namespace llvm { -class DomTreePrinterPass : public PassInfoMixin<DomTreePrinterPass> { -public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + +template <> +struct DOTGraphTraits<DomTreeNode *> : public DefaultDOTGraphTraits { + + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} + + std::string getNodeLabel(DomTreeNode *Node, DomTreeNode *Graph) { + + BasicBlock *BB = Node->getBlock(); + + if (!BB) + return "Post dominance root node"; + + if (isSimple()) + return DOTGraphTraits<DOTFuncInfo *>::getSimpleNodeLabel(BB, nullptr); + + return DOTGraphTraits<DOTFuncInfo *>::getCompleteNodeLabel(BB, nullptr); + } +}; + +template <> +struct DOTGraphTraits<DominatorTree *> + : public DOTGraphTraits<DomTreeNode *> { + + DOTGraphTraits(bool isSimple = false) + : DOTGraphTraits<DomTreeNode *>(isSimple) {} + + static std::string getGraphName(DominatorTree *DT) { + return "Dominator tree"; + } + + std::string getNodeLabel(DomTreeNode *Node, DominatorTree *G) { + return DOTGraphTraits<DomTreeNode *>::getNodeLabel(Node, + G->getRootNode()); + } +}; + +template<> +struct DOTGraphTraits<PostDominatorTree *> + : public DOTGraphTraits<DomTreeNode*> { + + DOTGraphTraits (bool isSimple=false) + : DOTGraphTraits<DomTreeNode*>(isSimple) {} + + static std::string getGraphName(PostDominatorTree *DT) { + return "Post dominator tree"; + } + + std::string getNodeLabel(DomTreeNode *Node, + PostDominatorTree *G) { + return DOTGraphTraits<DomTreeNode*>::getNodeLabel(Node, G->getRootNode()); + } +}; + +struct DomViewer final : DOTGraphTraitsViewer<DominatorTreeAnalysis, false> { + DomViewer() : DOTGraphTraitsViewer<DominatorTreeAnalysis, false>("dom") {} +}; + +struct DomOnlyViewer final : DOTGraphTraitsViewer<DominatorTreeAnalysis, true> { + DomOnlyViewer() + : DOTGraphTraitsViewer<DominatorTreeAnalysis, true>("domonly") {} +}; + +struct PostDomViewer final + : DOTGraphTraitsViewer<PostDominatorTreeAnalysis, false> { + PostDomViewer() + : DOTGraphTraitsViewer<PostDominatorTreeAnalysis, false>("postdom") {} +}; + +struct PostDomOnlyViewer final + : DOTGraphTraitsViewer<PostDominatorTreeAnalysis, true> { + PostDomOnlyViewer() + : DOTGraphTraitsViewer<PostDominatorTreeAnalysis, true>("postdomonly") {} +}; + +struct DomPrinter final : DOTGraphTraitsPrinter<DominatorTreeAnalysis, false> { + DomPrinter() : DOTGraphTraitsPrinter<DominatorTreeAnalysis, false>("dom") {} +}; + +struct DomOnlyPrinter final + : DOTGraphTraitsPrinter<DominatorTreeAnalysis, true> { + DomOnlyPrinter() + : DOTGraphTraitsPrinter<DominatorTreeAnalysis, true>("domonly") {} +}; + +struct PostDomPrinter final + : DOTGraphTraitsPrinter<PostDominatorTreeAnalysis, false> { + PostDomPrinter() + : DOTGraphTraitsPrinter<PostDominatorTreeAnalysis, false>("postdom") {} }; -class DomTreeOnlyPrinterPass : public PassInfoMixin<DomTreeOnlyPrinterPass> { -public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +struct PostDomOnlyPrinter final + : DOTGraphTraitsPrinter<PostDominatorTreeAnalysis, true> { + PostDomOnlyPrinter() + : DOTGraphTraitsPrinter<PostDominatorTreeAnalysis, true>("postdomonly") {} }; } // namespace llvm namespace llvm { class FunctionPass; - FunctionPass *createDomPrinterPass(); - FunctionPass *createDomOnlyPrinterPass(); - FunctionPass *createDomViewerPass(); - FunctionPass *createDomOnlyViewerPass(); - FunctionPass *createPostDomPrinterPass(); - FunctionPass *createPostDomOnlyPrinterPass(); - FunctionPass *createPostDomViewerPass(); - FunctionPass *createPostDomOnlyViewerPass(); + FunctionPass *createDomPrinterWrapperPassPass(); + FunctionPass *createDomOnlyPrinterWrapperPassPass(); + FunctionPass *createDomViewerWrapperPassPass(); + FunctionPass *createDomOnlyViewerWrapperPassPass(); + FunctionPass *createPostDomPrinterWrapperPassPass(); + FunctionPass *createPostDomOnlyPrinterWrapperPassPass(); + FunctionPass *createPostDomViewerWrapperPassPass(); + FunctionPass *createPostDomOnlyViewerWrapperPassPass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/Analysis/DomTreeUpdater.h b/llvm/include/llvm/Analysis/DomTreeUpdater.h index d09154d506ed..ddb958455ccd 100644 --- a/llvm/include/llvm/Analysis/DomTreeUpdater.h +++ b/llvm/include/llvm/Analysis/DomTreeUpdater.h @@ -150,49 +150,6 @@ public: /// awaiting deletion immediately. void recalculate(Function &F); - /// \deprecated { Submit an edge insertion to all available trees. The Eager - /// Strategy flushes this update immediately while the Lazy Strategy queues - /// the update. An internal function checks if the edge exists in the CFG in - /// DEBUG mode. CAUTION! This function has to be called *after* making the - /// update on the actual CFG. It is illegal to submit any update that has - /// already been applied. } - LLVM_ATTRIBUTE_DEPRECATED(void insertEdge(BasicBlock *From, BasicBlock *To), - "Use applyUpdates() instead."); - - /// \deprecated {Submit an edge insertion to all available trees. - /// Under either Strategy, an invalid update will be discard silently. - /// Invalid update means inserting an edge that does not exist in the CFG. - /// The Eager Strategy flushes this update immediately while the Lazy Strategy - /// queues the update. It is only recommended to use this method when you - /// want to discard an invalid update. - /// CAUTION! It is illegal to submit any update that has already been - /// submitted. } - LLVM_ATTRIBUTE_DEPRECATED(void insertEdgeRelaxed(BasicBlock *From, - BasicBlock *To), - "Use applyUpdatesPermissive() instead."); - - /// \deprecated { Submit an edge deletion to all available trees. The Eager - /// Strategy flushes this update immediately while the Lazy Strategy queues - /// the update. An internal function checks if the edge doesn't exist in the - /// CFG in DEBUG mode. - /// CAUTION! This function has to be called *after* making the update on the - /// actual CFG. It is illegal to submit any update that has already been - /// submitted. } - LLVM_ATTRIBUTE_DEPRECATED(void deleteEdge(BasicBlock *From, BasicBlock *To), - "Use applyUpdates() instead."); - - /// \deprecated { Submit an edge deletion to all available trees. - /// Under either Strategy, an invalid update will be discard silently. - /// Invalid update means deleting an edge that exists in the CFG. - /// The Eager Strategy flushes this update immediately while the Lazy Strategy - /// queues the update. It is only recommended to use this method when you - /// want to discard an invalid update. - /// CAUTION! It is illegal to submit any update that has already been - /// submitted. } - LLVM_ATTRIBUTE_DEPRECATED(void deleteEdgeRelaxed(BasicBlock *From, - BasicBlock *To), - "Use applyUpdatesPermissive() instead."); - /// Delete DelBB. DelBB will be removed from its Parent and /// erased from available trees if it exists and finally get deleted. /// Under Eager UpdateStrategy, DelBB will be processed immediately. diff --git a/llvm/include/llvm/Analysis/DominanceFrontierImpl.h b/llvm/include/llvm/Analysis/DominanceFrontierImpl.h index aa764be93b91..7a5f8f31bae3 100644 --- a/llvm/include/llvm/Analysis/DominanceFrontierImpl.h +++ b/llvm/include/llvm/Analysis/DominanceFrontierImpl.h @@ -17,7 +17,6 @@ #ifndef LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H #define LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H -#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Config/llvm-config.h" diff --git a/llvm/include/llvm/Analysis/EHPersonalities.h b/llvm/include/llvm/Analysis/EHPersonalities.h index eaada6627494..660d431bb063 100644 --- a/llvm/include/llvm/Analysis/EHPersonalities.h +++ b/llvm/include/llvm/Analysis/EHPersonalities.h @@ -11,7 +11,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/TinyPtrVector.h" -#include "llvm/Support/ErrorHandling.h" namespace llvm { class BasicBlock; diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h index cf07c873b17c..a0f5331fdba5 100644 --- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h +++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h @@ -14,16 +14,33 @@ #ifndef LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H #define LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" namespace llvm { +class DominatorTree; class Function; +class LoopInfo; class FunctionPropertiesInfo { + friend class FunctionPropertiesUpdater; + void updateForBB(const BasicBlock &BB, int64_t Direction); + void updateAggregateStats(const Function &F, const LoopInfo &LI); + void reIncludeBB(const BasicBlock &BB); + public: - static FunctionPropertiesInfo getFunctionPropertiesInfo(const Function &F, - const LoopInfo &LI); + static FunctionPropertiesInfo + getFunctionPropertiesInfo(const Function &F, FunctionAnalysisManager &FAM); + + bool operator==(const FunctionPropertiesInfo &FPI) const { + return std::memcmp(this, &FPI, sizeof(FunctionPropertiesInfo)) == 0; + } + + bool operator!=(const FunctionPropertiesInfo &FPI) const { + return !(*this == FPI); + } void print(raw_ostream &OS) const; @@ -57,6 +74,9 @@ public: // Number of Top Level Loops in the Function int64_t TopLevelLoopCount = 0; + + // All non-debug instructions + int64_t TotalInstructionCount = 0; }; // Analysis pass @@ -66,9 +86,9 @@ class FunctionPropertiesAnalysis public: static AnalysisKey Key; - using Result = FunctionPropertiesInfo; + using Result = const FunctionPropertiesInfo; - Result run(Function &F, FunctionAnalysisManager &FAM); + FunctionPropertiesInfo run(Function &F, FunctionAnalysisManager &FAM); }; /// Printer pass for the FunctionPropertiesAnalysis results. @@ -82,5 +102,24 @@ public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; +/// Correctly update FunctionPropertiesInfo post-inlining. A +/// FunctionPropertiesUpdater keeps the state necessary for tracking the changes +/// llvm::InlineFunction makes. The idea is that inlining will at most modify +/// a few BBs of the Caller (maybe the entry BB and definitely the callsite BB) +/// and potentially affect exception handling BBs in the case of invoke +/// inlining. +class FunctionPropertiesUpdater { +public: + FunctionPropertiesUpdater(FunctionPropertiesInfo &FPI, const CallBase &CB); + + void finish(FunctionAnalysisManager &FAM) const; + +private: + FunctionPropertiesInfo &FPI; + const BasicBlock &CallSiteBB; + const Function &Caller; + + DenseSet<const BasicBlock *> Successors; +}; } // namespace llvm #endif // LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H diff --git a/llvm/include/llvm/Analysis/GlobalsModRef.h b/llvm/include/llvm/Analysis/GlobalsModRef.h index 7daaa7f484de..4d8ed10bb18e 100644 --- a/llvm/include/llvm/Analysis/GlobalsModRef.h +++ b/llvm/include/llvm/Analysis/GlobalsModRef.h @@ -14,15 +14,14 @@ #define LLVM_ANALYSIS_GLOBALSMODREF_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include <list> namespace llvm { class CallGraph; +class Function; /// An alias analysis result set for globals. /// @@ -79,6 +78,8 @@ class GlobalsAAResult : public AAResultBase<GlobalsAAResult> { const DataLayout &DL, std::function<const TargetLibraryInfo &(Function &F)> GetTLI); + friend struct RecomputeGlobalsAAPass; + public: GlobalsAAResult(GlobalsAAResult &&Arg); ~GlobalsAAResult(); @@ -139,6 +140,10 @@ public: GlobalsAAResult run(Module &M, ModuleAnalysisManager &AM); }; +struct RecomputeGlobalsAAPass : PassInfoMixin<RecomputeGlobalsAAPass> { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + /// Legacy wrapper pass to provide the GlobalsAAResult object. class GlobalsAAWrapperPass : public ModulePass { std::unique_ptr<GlobalsAAResult> Result; diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h index 90ab2833e428..a3f1c1335cac 100644 --- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h +++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h @@ -51,12 +51,13 @@ #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/Allocator.h" namespace llvm { +class Module; + namespace IRSimilarity { struct IRInstructionDataList; @@ -546,7 +547,7 @@ struct IRInstructionMapper { // an outlined function. Also, assume-like intrinsics could be removed // from the region, removing arguments, causing discrepencies in the // number of inputs between different regions. - if (II.isLifetimeStartOrEnd() || II.isAssumeLikeIntrinsic()) + if (II.isAssumeLikeIntrinsic()) return Illegal; return EnableIntrinsics ? Legal : Illegal; } @@ -559,6 +560,18 @@ struct IRInstructionMapper { return Illegal; if (!F && !IsIndirectCall) return Illegal; + // Functions marked with the swifttailcc and tailcc calling conventions + // require special handling when outlining musttail functions. The + // calling convention must be passed down to the outlined function as + // well. Further, there is special handling for musttail calls as well, + // requiring a return call directly after. For now, the outliner does not + // support this, so we do not handle matching this case either. + if ((CI.getCallingConv() == CallingConv::SwiftTail || + CI.getCallingConv() == CallingConv::Tail) && + !EnableMustTailCalls) + return Illegal; + if (CI.isMustTailCall() && !EnableMustTailCalls) + return Illegal; return Legal; } // TODO: We do not current handle similarity that changes the control flow. @@ -580,6 +593,10 @@ struct IRInstructionMapper { // Flag that lets the classifier know whether we should allow intrinsics to // be checked for similarity. bool EnableIntrinsics = false; + + // Flag that lets the classifier know whether we should allow tail calls to + // be checked for similarity. + bool EnableMustTailCalls = false; }; /// Maps an Instruction to a member of InstrType. @@ -814,8 +831,6 @@ public: void getBasicBlocks(DenseSet<BasicBlock *> &BBSet) const { for (IRInstructionData &ID : *this) { BasicBlock *BB = ID.Inst->getParent(); - if (BBSet.contains(BB)) - continue; BBSet.insert(BB); } } @@ -826,10 +841,8 @@ public: SmallVector<BasicBlock *> &BBList) const { for (IRInstructionData &ID : *this) { BasicBlock *BB = ID.Inst->getParent(); - if (BBSet.contains(BB)) - continue; - BBSet.insert(BB); - BBList.push_back(BB); + if (BBSet.insert(BB).second) + BBList.push_back(BB); } } @@ -967,11 +980,13 @@ public: IRSimilarityIdentifier(bool MatchBranches = true, bool MatchIndirectCalls = true, bool MatchCallsWithName = false, - bool MatchIntrinsics = true) + bool MatchIntrinsics = true, + bool MatchMustTailCalls = true) : Mapper(&InstDataAllocator, &InstDataListAllocator), EnableBranches(MatchBranches), EnableIndirectCalls(MatchIndirectCalls), EnableMatchingCallsByName(MatchCallsWithName), - EnableIntrinsics(MatchIntrinsics) {} + EnableIntrinsics(MatchIntrinsics), + EnableMustTailCalls(MatchMustTailCalls) {} private: /// Map the instructions in the module to unsigned integers, using mapping @@ -1024,7 +1039,7 @@ public: // If we've already analyzed a Module or set of Modules, so we must clear // the SimilarityCandidates to make sure we do not have only old values // hanging around. - if (SimilarityCandidates.hasValue()) + if (SimilarityCandidates) SimilarityCandidates->clear(); else SimilarityCandidates = SimilarityGroupList(); @@ -1064,6 +1079,10 @@ private: /// similarity. bool EnableIntrinsics = true; + // The flag variable that marks whether we should allow tailcalls + // to be checked for similarity. + bool EnableMustTailCalls = false; + /// The SimilarityGroups found with the most recent run of \ref /// findSimilarity. None if there is no recent run. Optional<SimilarityGroupList> SimilarityCandidates; diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index dec488a6f26d..231d3bbf534b 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -13,27 +13,23 @@ #ifndef LLVM_ANALYSIS_IVDESCRIPTORS_H #define LLVM_ANALYSIS_IVDESCRIPTORS_H -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Casting.h" namespace llvm { -class DemandedBits; class AssumptionCache; +class DemandedBits; +class DominatorTree; +class Instruction; class Loop; class PredicatedScalarEvolution; class ScalarEvolution; class SCEV; -class DominatorTree; +class StoreInst; /// These are the kinds of recurrences that we support. enum class RecurKind { @@ -74,14 +70,14 @@ class RecurrenceDescriptor { public: RecurrenceDescriptor() = default; - RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurKind K, - FastMathFlags FMF, Instruction *ExactFP, Type *RT, - bool Signed, bool Ordered, + RecurrenceDescriptor(Value *Start, Instruction *Exit, StoreInst *Store, + RecurKind K, FastMathFlags FMF, Instruction *ExactFP, + Type *RT, bool Signed, bool Ordered, SmallPtrSetImpl<Instruction *> &CI, unsigned MinWidthCastToRecurTy) - : StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF), - ExactFPMathInst(ExactFP), RecurrenceType(RT), IsSigned(Signed), - IsOrdered(Ordered), + : IntermediateStore(Store), StartValue(Start), LoopExitInstr(Exit), + Kind(K), FMF(FMF), ExactFPMathInst(ExactFP), RecurrenceType(RT), + IsSigned(Signed), IsOrdered(Ordered), MinWidthCastToRecurrenceType(MinWidthCastToRecurTy) { CastInsts.insert(CI.begin(), CI.end()); } @@ -168,22 +164,21 @@ public: /// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are /// non-null, the minimal bit width needed to compute the reduction will be /// computed. - static bool AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop, - FastMathFlags FuncFMF, - RecurrenceDescriptor &RedDes, - DemandedBits *DB = nullptr, - AssumptionCache *AC = nullptr, - DominatorTree *DT = nullptr); + static bool + AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop, + FastMathFlags FuncFMF, RecurrenceDescriptor &RedDes, + DemandedBits *DB = nullptr, AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr, ScalarEvolution *SE = nullptr); /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor /// is returned in RedDes. If either \p DB is non-null or \p AC and \p DT are /// non-null, the minimal bit width needed to compute the reduction will be - /// computed. - static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, - RecurrenceDescriptor &RedDes, - DemandedBits *DB = nullptr, - AssumptionCache *AC = nullptr, - DominatorTree *DT = nullptr); + /// computed. If \p SE is non-null, store instructions to loop invariant + /// addresses are processed. + static bool + isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes, + DemandedBits *DB = nullptr, AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr, ScalarEvolution *SE = nullptr); /// Returns true if Phi is a first-order recurrence. A first-order recurrence /// is a non-reduction recurrence relation in which the value of the @@ -275,6 +270,11 @@ public: cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::fmuladd; } + /// Reductions may store temporary or final result to an invariant address. + /// If there is such a store in the loop then, after successfull run of + /// AddReductionVar method, this field will be assigned the last met store. + StoreInst *IntermediateStore = nullptr; + private: // The starting value of the recurrence. // It does not have to be zero! diff --git a/llvm/include/llvm/Analysis/IVUsers.h b/llvm/include/llvm/Analysis/IVUsers.h index 390d09848dde..e5a496037691 100644 --- a/llvm/include/llvm/Analysis/IVUsers.h +++ b/llvm/include/llvm/Analysis/IVUsers.h @@ -23,8 +23,6 @@ namespace llvm { class AssumptionCache; class DominatorTree; -class Instruction; -class Value; class ScalarEvolution; class SCEV; class IVUsers; diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h index 0103ee7f8386..31524126027b 100644 --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -9,19 +9,20 @@ #ifndef LLVM_ANALYSIS_INLINEADVISOR_H #define LLVM_ANALYSIS_INLINEADVISOR_H +#include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/LazyCallGraph.h" -#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/PassManager.h" #include <memory> -#include <unordered_set> namespace llvm { class BasicBlock; class CallBase; class Function; class Module; +class OptimizationRemark; +class ImportedFunctionsInliningStatistics; class OptimizationRemarkEmitter; struct ReplayInlinerSettings; @@ -40,6 +41,28 @@ struct ReplayInlinerSettings; /// training. enum class InliningAdvisorMode : int { Default, Release, Development }; +// Each entry represents an inline driver. +enum class InlinePass : int { + AlwaysInliner, + CGSCCInliner, + EarlyInliner, + ModuleInliner, + MLInliner, + ReplayCGSCCInliner, + ReplaySampleProfileInliner, + SampleProfileInliner, +}; + +/// Provides context on when an inline advisor is constructed in the pipeline +/// (e.g., link phase, inline driver). +struct InlineContext { + ThinOrFullLTOPhase LTOPhase; + + InlinePass Pass; +}; + +std::string AnnotateInlinePassName(InlineContext IC); + class InlineAdvisor; /// Capture state between an inlining decision having had been made, and /// its impact being observable. When collecting model training data, this @@ -122,7 +145,7 @@ public: DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE, bool EmitRemarks = true) - : InlineAdvice(Advisor, CB, ORE, OIC.hasValue()), OriginalCB(&CB), + : InlineAdvice(Advisor, CB, ORE, OIC.has_value()), OriginalCB(&CB), OIC(OIC), EmitRemarks(EmitRemarks) {} private: @@ -158,7 +181,7 @@ public: /// This must be called when the Inliner pass is entered, to allow the /// InlineAdvisor update internal state, as result of function passes run /// between Inliner pass runs (for the same module). - virtual void onPassEntry() {} + virtual void onPassEntry(LazyCallGraph::SCC *SCC = nullptr) {} /// This must be called when the Inliner pass is exited, as function passes /// may be run subsequently. This allows an implementation of InlineAdvisor @@ -170,14 +193,22 @@ public: OS << "Unimplemented InlineAdvisor print\n"; } + /// NOTE pass name is annotated only when inline advisor constructor provides InlineContext. + const char *getAnnotatedInlinePassName() const { + return AnnotatedInlinePassName.c_str(); + } + protected: - InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); + InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + Optional<InlineContext> IC = NoneType::None); virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, bool Advice); Module &M; FunctionAnalysisManager &FAM; + const Optional<InlineContext> IC; + const std::string AnnotatedInlinePassName; std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; enum class MandatoryInliningKind { NotMandatory, Always, Never }; @@ -198,8 +229,8 @@ private: class DefaultInlineAdvisor : public InlineAdvisor { public: DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, - InlineParams Params) - : InlineAdvisor(M, FAM), Params(Params) {} + InlineParams Params, InlineContext IC) + : InlineAdvisor(M, FAM, IC), Params(Params) {} private: std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; @@ -223,7 +254,8 @@ public: return !PAC.preservedWhenStateless(); } bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, - const ReplayInlinerSettings &ReplaySettings); + const ReplayInlinerSettings &ReplaySettings, + InlineContext IC); InlineAdvisor *getAdvisor() const { return Advisor.get(); } private: @@ -244,6 +276,9 @@ public: explicit InlineAdvisorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); + + PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, + LazyCallGraph &CG, CGSCCUpdateResult &UR); }; std::unique_ptr<InlineAdvisor> diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index f86ee5a14874..756f1fb61f95 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -13,14 +13,17 @@ #ifndef LLVM_ANALYSIS_INLINECOST_H #define LLVM_ANALYSIS_INLINECOST_H -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Analysis/InlineModelFeatureMaps.h" -#include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/IR/PassManager.h" #include <cassert> #include <climits> namespace llvm { +class AssumptionCache; +class OptimizationRemarkEmitter; class BlockFrequencyInfo; class CallBase; class DataLayout; @@ -52,6 +55,9 @@ const unsigned TotalAllocaSizeRecursiveCaller = 1024; /// Do not inline dynamic allocas that have been constant propagated to be /// static allocas above this amount in bytes. const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536; + +const char FunctionInlineCostMultiplierAttributeName[] = + "function-inline-cost-multiplier"; } // namespace InlineConstants // The cost-benefit pair computed by cost-benefit analysis. @@ -217,6 +223,8 @@ struct InlineParams { Optional<bool> AllowRecursiveCall = false; }; +Optional<int> getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind); + /// Generate the parameters to tune the inline cost analysis based only on the /// commandline options. InlineParams getInlineParams(); diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h index 1afa8a825f15..fb8236c28b25 100644 --- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h +++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h @@ -10,6 +10,8 @@ #ifndef LLVM_ANALYSIS_INLINEMODELFEATUREMAPS_H #define LLVM_ANALYSIS_INLINEMODELFEATUREMAPS_H +#include "llvm/Analysis/TensorSpec.h" + #include <array> #include <string> #include <vector> @@ -127,7 +129,7 @@ inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature) { constexpr size_t NumberOfFeatures = static_cast<size_t>(FeatureIndex::NumberOfFeatures); -extern const std::array<std::string, NumberOfFeatures> FeatureNameMap; +extern const std::array<TensorSpec, NumberOfFeatures> FeatureMap; extern const char *const DecisionName; extern const char *const DefaultDecisionName; diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h index 84252bcf1b06..aabd86c98780 100644 --- a/llvm/include/llvm/Analysis/InlineOrder.h +++ b/llvm/include/llvm/Analysis/InlineOrder.h @@ -10,10 +10,9 @@ #define LLVM_ANALYSIS_INLINEORDER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instruction.h" -#include "llvm/IR/Instructions.h" +#include "llvm/IR/InstrTypes.h" #include <algorithm> #include <utility> @@ -71,34 +70,52 @@ private: size_t FirstIndex = 0; }; -class InlineSizePriority { +class InlinePriority { public: - InlineSizePriority(int Size) : Size(Size) {} + virtual ~InlinePriority() = default; + virtual bool hasLowerPriority(const CallBase *L, const CallBase *R) const = 0; + virtual void update(const CallBase *CB) = 0; + virtual bool updateAndCheckDecreased(const CallBase *CB) = 0; +}; - static bool isMoreDesirable(const InlineSizePriority &S1, - const InlineSizePriority &S2) { - return S1.Size < S2.Size; - } +class SizePriority : public InlinePriority { + using PriorityT = unsigned; + DenseMap<const CallBase *, PriorityT> Priorities; - static InlineSizePriority evaluate(CallBase *CB) { + static PriorityT evaluate(const CallBase *CB) { Function *Callee = CB->getCalledFunction(); - return InlineSizePriority(Callee->getInstructionCount()); + return Callee->getInstructionCount(); + } + + static bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) { + return P1 < P2; } - int Size; + bool hasLowerPriority(const CallBase *L, const CallBase *R) const override { + const auto I1 = Priorities.find(L); + const auto I2 = Priorities.find(R); + assert(I1 != Priorities.end() && I2 != Priorities.end()); + return isMoreDesirable(I2->second, I1->second); + } + +public: + // Update the priority associated with CB. + void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); }; + + bool updateAndCheckDecreased(const CallBase *CB) override { + auto It = Priorities.find(CB); + const auto OldPriority = It->second; + It->second = evaluate(CB); + const auto NewPriority = It->second; + return isMoreDesirable(OldPriority, NewPriority); + } }; -template <typename PriorityT> class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> { using T = std::pair<CallBase *, int>; - using HeapT = std::pair<CallBase *, PriorityT>; using reference = T &; using const_reference = const T &; - static bool cmp(const HeapT &P1, const HeapT &P2) { - return PriorityT::isMoreDesirable(P2.second, P1.second); - } - // A call site could become less desirable for inlining because of the size // growth from prior inlining into the callee. This method is used to lazily // update the desirability of a call site if it's decreasing. It is only @@ -107,31 +124,29 @@ class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> { // pushed right back into the heap. For simplicity, those cases where // the desirability of a call site increases are ignored here. void adjust() { - bool Changed = false; - do { - CallBase *CB = Heap.front().first; - const PriorityT PreviousGoodness = Heap.front().second; - const PriorityT CurrentGoodness = PriorityT::evaluate(CB); - Changed = PriorityT::isMoreDesirable(PreviousGoodness, CurrentGoodness); - if (Changed) { - std::pop_heap(Heap.begin(), Heap.end(), cmp); - Heap.pop_back(); - Heap.push_back({CB, CurrentGoodness}); - std::push_heap(Heap.begin(), Heap.end(), cmp); - } - } while (Changed); + while (PriorityPtr->updateAndCheckDecreased(Heap.front())) { + std::pop_heap(Heap.begin(), Heap.end(), isLess); + std::push_heap(Heap.begin(), Heap.end(), isLess); + } } public: + PriorityInlineOrder(std::unique_ptr<InlinePriority> PriorityPtr) + : PriorityPtr(std::move(PriorityPtr)) { + isLess = [this](const CallBase *L, const CallBase *R) { + return this->PriorityPtr->hasLowerPriority(L, R); + }; + } + size_t size() override { return Heap.size(); } void push(const T &Elt) override { CallBase *CB = Elt.first; const int InlineHistoryID = Elt.second; - const PriorityT Goodness = PriorityT::evaluate(CB); - Heap.push_back({CB, Goodness}); - std::push_heap(Heap.begin(), Heap.end(), cmp); + Heap.push_back(CB); + PriorityPtr->update(CB); + std::push_heap(Heap.begin(), Heap.end(), isLess); InlineHistoryMap[CB] = InlineHistoryID; } @@ -139,10 +154,10 @@ public: assert(size() > 0); adjust(); - CallBase *CB = Heap.front().first; + CallBase *CB = Heap.front(); T Result = std::make_pair(CB, InlineHistoryMap[CB]); InlineHistoryMap.erase(CB); - std::pop_heap(Heap.begin(), Heap.end(), cmp); + std::pop_heap(Heap.begin(), Heap.end(), isLess); Heap.pop_back(); return Result; } @@ -151,21 +166,23 @@ public: assert(size() > 0); adjust(); - CallBase *CB = Heap.front().first; + CallBase *CB = Heap.front(); return *InlineHistoryMap.find(CB); } void erase_if(function_ref<bool(T)> Pred) override { - auto PredWrapper = [=](HeapT P) -> bool { - return Pred(std::make_pair(P.first, 0)); + auto PredWrapper = [=](CallBase *CB) -> bool { + return Pred(std::make_pair(CB, 0)); }; llvm::erase_if(Heap, PredWrapper); - std::make_heap(Heap.begin(), Heap.end(), cmp); + std::make_heap(Heap.begin(), Heap.end(), isLess); } private: - SmallVector<HeapT, 16> Heap; + SmallVector<CallBase *, 16> Heap; + std::function<bool(const CallBase *L, const CallBase *R)> isLess; DenseMap<CallBase *, int> InlineHistoryMap; + std::unique_ptr<InlinePriority> PriorityPtr; }; } // namespace llvm #endif // LLVM_ANALYSIS_INLINEORDER_H diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h index 54ef1ddf6085..d4ea7d73ec92 100644 --- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h +++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h @@ -22,12 +22,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/TargetFolder.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilderFolder.h" -#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" namespace llvm { +class Constant; /// InstSimplifyFolder - Use InstructionSimplify to fold operations to existing /// values. Also applies target-specific constant folding when not using @@ -47,108 +46,74 @@ public: // Return an existing value or a constant if the operation can be simplified. // Otherwise return nullptr. //===--------------------------------------------------------------------===// - Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - return SimplifyAddInst(LHS, RHS, HasNUW, HasNSW, SQ); + + Value *FoldBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS) const override { + return simplifyBinOp(Opc, LHS, RHS, SQ); + } + + Value *FoldExactBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool IsExact) const override { + return simplifyBinOp(Opc, LHS, RHS, SQ); } - Value *FoldAnd(Value *LHS, Value *RHS) const override { - return SimplifyAndInst(LHS, RHS, SQ); + Value *FoldNoWrapBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool HasNUW, bool HasNSW) const override { + return simplifyBinOp(Opc, LHS, RHS, SQ); } - Value *FoldOr(Value *LHS, Value *RHS) const override { - return SimplifyOrInst(LHS, RHS, SQ); + Value *FoldBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + FastMathFlags FMF) const override { + return simplifyBinOp(Opc, LHS, RHS, FMF, SQ); } Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { - return SimplifyICmpInst(P, LHS, RHS, SQ); + return simplifyICmpInst(P, LHS, RHS, SQ); } Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, bool IsInBounds = false) const override { - return SimplifyGEPInst(Ty, Ptr, IdxList, IsInBounds, SQ); + return simplifyGEPInst(Ty, Ptr, IdxList, IsInBounds, SQ); } Value *FoldSelect(Value *C, Value *True, Value *False) const override { - return SimplifySelectInst(C, True, False, SQ); + return simplifySelectInst(C, True, False, SQ); } - //===--------------------------------------------------------------------===// - // Binary Operators - //===--------------------------------------------------------------------===// + Value *FoldExtractValue(Value *Agg, + ArrayRef<unsigned> IdxList) const override { + return simplifyExtractValueInst(Agg, IdxList, SQ); + }; - Value *CreateFAdd(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateFAdd(LHS, RHS); - } - Value *CreateSub(Constant *LHS, Constant *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - return ConstFolder.CreateSub(LHS, RHS, HasNUW, HasNSW); - } - Value *CreateFSub(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateFSub(LHS, RHS); - } - Value *CreateMul(Constant *LHS, Constant *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - return ConstFolder.CreateMul(LHS, RHS, HasNUW, HasNSW); - } - Value *CreateFMul(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateFMul(LHS, RHS); - } - Value *CreateUDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstFolder.CreateUDiv(LHS, RHS, isExact); - } - Value *CreateSDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstFolder.CreateSDiv(LHS, RHS, isExact); - } - Value *CreateFDiv(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateFDiv(LHS, RHS); - } - Value *CreateURem(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateURem(LHS, RHS); - } - Value *CreateSRem(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateSRem(LHS, RHS); - } - Value *CreateFRem(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateFRem(LHS, RHS); - } - Value *CreateShl(Constant *LHS, Constant *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - return ConstFolder.CreateShl(LHS, RHS, HasNUW, HasNSW); - } - Value *CreateLShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstFolder.CreateLShr(LHS, RHS, isExact); + Value *FoldInsertValue(Value *Agg, Value *Val, + ArrayRef<unsigned> IdxList) const override { + return simplifyInsertValueInst(Agg, Val, IdxList, SQ); } - Value *CreateAShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstFolder.CreateAShr(LHS, RHS, isExact); + + Value *FoldExtractElement(Value *Vec, Value *Idx) const override { + return simplifyExtractElementInst(Vec, Idx, SQ); } - Value *CreateXor(Constant *LHS, Constant *RHS) const override { - return ConstFolder.CreateXor(LHS, RHS); + + Value *FoldInsertElement(Value *Vec, Value *NewElt, + Value *Idx) const override { + return simplifyInsertElementInst(Vec, NewElt, Idx, SQ); } - Value *CreateBinOp(Instruction::BinaryOps Opc, Constant *LHS, - Constant *RHS) const override { - return ConstFolder.CreateBinOp(Opc, LHS, RHS); + Value *FoldShuffleVector(Value *V1, Value *V2, + ArrayRef<int> Mask) const override { + Type *RetTy = VectorType::get( + cast<VectorType>(V1->getType())->getElementType(), Mask.size(), + isa<ScalableVectorType>(V1->getType())); + return simplifyShuffleVectorInst(V1, V2, Mask, RetTy, SQ); } //===--------------------------------------------------------------------===// // Unary Operators //===--------------------------------------------------------------------===// - Value *CreateNeg(Constant *C, bool HasNUW = false, - bool HasNSW = false) const override { - return ConstFolder.CreateNeg(C, HasNUW, HasNSW); - } Value *CreateFNeg(Constant *C) const override { return ConstFolder.CreateFNeg(C); } - Value *CreateNot(Constant *C) const override { - return ConstFolder.CreateNot(C); - } Value *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const override { return ConstFolder.CreateUnOp(Opc, C); @@ -220,34 +185,6 @@ public: Constant *RHS) const override { return ConstFolder.CreateFCmp(P, LHS, RHS); } - - //===--------------------------------------------------------------------===// - // Other Instructions - //===--------------------------------------------------------------------===// - - Value *CreateExtractElement(Constant *Vec, Constant *Idx) const override { - return ConstFolder.CreateExtractElement(Vec, Idx); - } - - Value *CreateInsertElement(Constant *Vec, Constant *NewElt, - Constant *Idx) const override { - return ConstFolder.CreateInsertElement(Vec, NewElt, Idx); - } - - Value *CreateShuffleVector(Constant *V1, Constant *V2, - ArrayRef<int> Mask) const override { - return ConstFolder.CreateShuffleVector(V1, V2, Mask); - } - - Value *CreateExtractValue(Constant *Agg, - ArrayRef<unsigned> IdxList) const override { - return ConstFolder.CreateExtractValue(Agg, IdxList); - } - - Value *CreateInsertValue(Constant *Agg, Constant *Val, - ArrayRef<unsigned> IdxList) const override { - return ConstFolder.CreateInsertValue(Agg, Val, IdxList); - } }; } // end namespace llvm diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h index 8b49c115f101..52d43bf5c2a6 100644 --- a/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -35,8 +35,6 @@ #ifndef LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H #define LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H -#include "llvm/IR/Instruction.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" namespace llvm { @@ -49,6 +47,7 @@ class CallBase; class DataLayout; class DominatorTree; class Function; +class Instruction; struct LoopStandardAnalysisResults; class MDNode; class OptimizationRemarkEmitter; @@ -145,176 +144,185 @@ struct SimplifyQuery { // Please use the SimplifyQuery versions in new code. /// Given operand for an FNeg, fold the result or return null. -Value *SimplifyFNegInst(Value *Op, FastMathFlags FMF, const SimplifyQuery &Q); +Value *simplifyFNegInst(Value *Op, FastMathFlags FMF, const SimplifyQuery &Q); /// Given operands for an Add, fold the result or return null. -Value *SimplifyAddInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, +Value *simplifyAddInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, const SimplifyQuery &Q); /// Given operands for a Sub, fold the result or return null. -Value *SimplifySubInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, +Value *simplifySubInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, const SimplifyQuery &Q); /// Given operands for an FAdd, fold the result or return null. Value * -SimplifyFAddInst(Value *LHS, Value *RHS, FastMathFlags FMF, +simplifyFAddInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for an FSub, fold the result or return null. Value * -SimplifyFSubInst(Value *LHS, Value *RHS, FastMathFlags FMF, +simplifyFSubInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for an FMul, fold the result or return null. Value * -SimplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, +simplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for the multiplication of a FMA, fold the result or return -/// null. In contrast to SimplifyFMulInst, this function will not perform +/// null. In contrast to simplifyFMulInst, this function will not perform /// simplifications whose unrounded results differ when rounded to the argument /// type. -Value *SimplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, +Value *simplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for a Mul, fold the result or return null. -Value *SimplifyMulInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyMulInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an SDiv, fold the result or return null. -Value *SimplifySDivInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifySDivInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for a UDiv, fold the result or return null. -Value *SimplifyUDivInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyUDivInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an FDiv, fold the result or return null. Value * -SimplifyFDivInst(Value *LHS, Value *RHS, FastMathFlags FMF, +simplifyFDivInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for an SRem, fold the result or return null. -Value *SimplifySRemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifySRemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for a URem, fold the result or return null. -Value *SimplifyURemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyURemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an FRem, fold the result or return null. Value * -SimplifyFRemInst(Value *LHS, Value *RHS, FastMathFlags FMF, +simplifyFRemInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven); /// Given operands for a Shl, fold the result or return null. -Value *SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, +Value *simplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const SimplifyQuery &Q); /// Given operands for a LShr, fold the result or return null. -Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, +Value *simplifyLShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q); /// Given operands for a AShr, fold the result or return nulll. -Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, +Value *simplifyAShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q); /// Given operands for an And, fold the result or return null. -Value *SimplifyAndInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyAndInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an Or, fold the result or return null. -Value *SimplifyOrInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyOrInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an Xor, fold the result or return null. -Value *SimplifyXorInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); +Value *simplifyXorInst(Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an ICmpInst, fold the result or return null. -Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for an FCmpInst, fold the result or return null. -Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q); /// Given operands for a SelectInst, fold the result or return null. -Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, +Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const SimplifyQuery &Q); /// Given operands for a GetElementPtrInst, fold the result or return null. -Value *SimplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, +Value *simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, bool InBounds, const SimplifyQuery &Q); /// Given operands for an InsertValueInst, fold the result or return null. -Value *SimplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, +Value *simplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q); /// Given operands for an InsertElement, fold the result or return null. -Value *SimplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, +Value *simplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, const SimplifyQuery &Q); /// Given operands for an ExtractValueInst, fold the result or return null. -Value *SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, +Value *simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q); /// Given operands for an ExtractElementInst, fold the result or return null. -Value *SimplifyExtractElementInst(Value *Vec, Value *Idx, +Value *simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q); /// Given operands for a CastInst, fold the result or return null. -Value *SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, +Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, const SimplifyQuery &Q); /// Given operands for a ShuffleVectorInst, fold the result or return null. /// See class ShuffleVectorInst for a description of the mask representation. -Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask, +Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask, Type *RetTy, const SimplifyQuery &Q); //=== Helper functions for higher up the class hierarchy. /// Given operands for a CmpInst, fold the result or return null. -Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operand for a UnaryOperator, fold the result or return null. -Value *SimplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q); +Value *simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q); /// Given operand for a UnaryOperator, fold the result or return null. /// Try to use FastMathFlags when folding the result. -Value *SimplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, +Value *simplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, const SimplifyQuery &Q); /// Given operands for a BinaryOperator, fold the result or return null. -Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, +Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q); /// Given operands for a BinaryOperator, fold the result or return null. /// Try to use FastMathFlags when folding the result. -Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, FastMathFlags FMF, +Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q); /// Given a callsite, fold the result or return null. -Value *SimplifyCall(CallBase *Call, const SimplifyQuery &Q); +Value *simplifyCall(CallBase *Call, const SimplifyQuery &Q); + +/// Given a constrained FP intrinsic call, tries to compute its simplified +/// version. Returns a simplified result or null. +/// +/// This function provides an additional contract: it guarantees that if +/// simplification succeeds that the intrinsic is side effect free. As a result, +/// successful simplification can be used to delete the intrinsic not just +/// replace its result. +Value *simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q); /// Given an operand for a Freeze, see if we can fold the result. /// If not, this returns null. -Value *SimplifyFreezeInst(Value *Op, const SimplifyQuery &Q); +Value *simplifyFreezeInst(Value *Op, const SimplifyQuery &Q); /// See if we can compute a simplified version of this instruction. If not, /// return null. -Value *SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, +Value *simplifyInstruction(Instruction *I, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE = nullptr); -/// Like \p SimplifyInstruction but the operands of \p I are replaced with +/// Like \p simplifyInstruction but the operands of \p I are replaced with /// \p NewOps. Returns a simplified value, or null if none was found. Value * -SimplifyInstructionWithOperands(Instruction *I, ArrayRef<Value *> NewOps, +simplifyInstructionWithOperands(Instruction *I, ArrayRef<Value *> NewOps, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE = nullptr); diff --git a/llvm/include/llvm/Analysis/IntervalIterator.h b/llvm/include/llvm/Analysis/IntervalIterator.h index 8e2273618a66..cbb7cac1c508 100644 --- a/llvm/include/llvm/Analysis/IntervalIterator.h +++ b/llvm/include/llvm/Analysis/IntervalIterator.h @@ -36,8 +36,6 @@ #include "llvm/Analysis/Interval.h" #include "llvm/Analysis/IntervalPartition.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/Function.h" -#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <cassert> #include <iterator> @@ -48,6 +46,7 @@ namespace llvm { class BasicBlock; +class Function; // getNodeHeader - Given a source graph node and the source graph, return the // BasicBlock that is the header node. This is the opposite of diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h index c0404d37d04d..4cacf8951d6a 100644 --- a/llvm/include/llvm/Analysis/LazyCallGraph.h +++ b/llvm/include/llvm/Analysis/LazyCallGraph.h @@ -38,20 +38,14 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerIntPair.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/Constant.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <iterator> @@ -60,8 +54,11 @@ namespace llvm { +class Constant; +class Function; template <class GraphType> struct GraphTraits; class Module; +class TargetLibraryInfo; class Value; /// A lazily constructed view of the call graph of a module. @@ -331,7 +328,7 @@ public: bool operator!=(const Node &N) const { return !operator==(N); } /// Tests whether the node has been populated with edges. - bool isPopulated() const { return Edges.hasValue(); } + bool isPopulated() const { return Edges.has_value(); } /// Tests whether this is actually a dead node and no longer valid. /// diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h index 754391e10630..24c2bfcc74b9 100644 --- a/llvm/include/llvm/Analysis/LazyValueInfo.h +++ b/llvm/include/llvm/Analysis/LazyValueInfo.h @@ -114,6 +114,9 @@ public: /// Inform the analysis cache that we have erased a block. void eraseBlock(BasicBlock *BB); + /// Complete flush all previously computed values + void clear(const Module *M); + /// Print the \LazyValueInfo Analysis. /// We pass in the DTree that is required for identifying which basic blocks /// we can solve/print for, in the LVIPrinter. diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h index 09bf98d324ed..29e3efb38e19 100644 --- a/llvm/include/llvm/Analysis/Loads.h +++ b/llvm/include/llvm/Analysis/Loads.h @@ -75,9 +75,9 @@ bool isSafeToLoadUnconditionally(Value *V, Align Alignment, APInt &Size, /// within the specified loop) would access only dereferenceable memory, and /// be properly aligned on every iteration of the specified loop regardless of /// its placement within the loop. (i.e. does not require predication beyond -/// that required by the the header itself and could be hoisted into the header +/// that required by the header itself and could be hoisted into the header /// if desired.) This is more powerful than the variants above when the -/// address loaded from is analyzeable by SCEV. +/// address loaded from is analyzeable by SCEV. bool isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L, ScalarEvolution &SE, DominatorTree &DT); diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index c83a04991b04..8f71ce9e96c0 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -244,6 +244,15 @@ public: SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr, bool isWrite) const; + /// Return the program order indices for the access location (Ptr, IsWrite). + /// Returns an empty ArrayRef if there are no accesses for the location. + ArrayRef<unsigned> getOrderForAccess(Value *Ptr, bool IsWrite) const { + auto I = Accesses.find({Ptr, IsWrite}); + if (I != Accesses.end()) + return I->second; + return {}; + } + private: /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and /// applies dynamic knowledge to simplify SCEV expressions and convert them @@ -327,12 +336,6 @@ struct RuntimeCheckingPtrGroup { /// pointer, with index \p Index in RtCheck. RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck); - RuntimeCheckingPtrGroup(unsigned Index, const SCEV *Start, const SCEV *End, - unsigned AS) - : High(End), Low(Start), AddressSpace(AS) { - Members.push_back(Index); - } - /// Tries to add the pointer recorded in RtCheck at index /// \p Index to this pointer checking group. We can only add a pointer /// to a checking group if we will still be able to get @@ -340,7 +343,7 @@ struct RuntimeCheckingPtrGroup { /// of success, false otherwise. bool addPointer(unsigned Index, RuntimePointerChecking &RtCheck); bool addPointer(unsigned Index, const SCEV *Start, const SCEV *End, - unsigned AS, ScalarEvolution &SE); + unsigned AS, bool NeedsFreeze, ScalarEvolution &SE); /// The SCEV expression which represents the upper bound of all the /// pointers in this group. @@ -352,6 +355,9 @@ struct RuntimeCheckingPtrGroup { SmallVector<unsigned, 2> Members; /// Address space of the involved pointers. unsigned AddressSpace; + /// Whether the pointer needs to be frozen after expansion, e.g. because it + /// may be poison outside the loop. + bool NeedsFreeze = false; }; /// A memcheck which made up of a pair of grouped pointers. @@ -359,6 +365,18 @@ typedef std::pair<const RuntimeCheckingPtrGroup *, const RuntimeCheckingPtrGroup *> RuntimePointerCheck; +struct PointerDiffInfo { + const SCEV *SrcStart; + const SCEV *SinkStart; + unsigned AccessSize; + bool NeedsFreeze; + + PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart, + unsigned AccessSize, bool NeedsFreeze) + : SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize), + NeedsFreeze(NeedsFreeze) {} +}; + /// Holds information about the memory runtime legality checks to verify /// that a group of pointers do not overlap. class RuntimePointerChecking { @@ -383,16 +401,19 @@ public: unsigned AliasSetId; /// SCEV for the access. const SCEV *Expr; + /// True if the pointer expressions needs to be frozen after expansion. + bool NeedsFreeze; PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End, bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId, - const SCEV *Expr) + const SCEV *Expr, bool NeedsFreeze) : PointerValue(PointerValue), Start(Start), End(End), IsWritePtr(IsWritePtr), DependencySetId(DependencySetId), - AliasSetId(AliasSetId), Expr(Expr) {} + AliasSetId(AliasSetId), Expr(Expr), NeedsFreeze(NeedsFreeze) {} }; - RuntimePointerChecking(ScalarEvolution *SE) : SE(SE) {} + RuntimePointerChecking(MemoryDepChecker &DC, ScalarEvolution *SE) + : DC(DC), SE(SE) {} /// Reset the state of the pointer runtime information. void reset() { @@ -406,9 +427,9 @@ public: /// according to the assumptions that we've made during the analysis. /// The method might also version the pointer stride according to \p Strides, /// and add new predicates to \p PSE. - void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId, - unsigned ASId, const ValueToValueMap &Strides, - PredicatedScalarEvolution &PSE); + void insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, Type *AccessTy, + bool WritePtr, unsigned DepSetId, unsigned ASId, + PredicatedScalarEvolution &PSE, bool NeedsFreeze); /// No run-time memory checking is necessary. bool empty() const { return Pointers.empty(); } @@ -418,11 +439,23 @@ public: void generateChecks(MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies); - /// Returns the checks that generateChecks created. + /// Returns the checks that generateChecks created. They can be used to ensure + /// no read/write accesses overlap across all loop iterations. const SmallVectorImpl<RuntimePointerCheck> &getChecks() const { return Checks; } + // Returns an optional list of (pointer-difference expressions, access size) + // pairs that can be used to prove that there are no vectorization-preventing + // dependencies at runtime. There are is a vectorization-preventing dependency + // if any pointer-difference is <u VF * InterleaveCount * access size. Returns + // None if pointer-difference checks cannot be used. + Optional<ArrayRef<PointerDiffInfo>> getDiffChecks() const { + if (!CanUseDiffCheck) + return None; + return {DiffChecks}; + } + /// Decide if we need to add a check between two groups of pointers, /// according to needsChecking. bool needsChecking(const RuntimeCheckingPtrGroup &M, @@ -477,7 +510,15 @@ private: bool UseDependencies); /// Generate the checks and return them. - SmallVector<RuntimePointerCheck, 4> generateChecks() const; + SmallVector<RuntimePointerCheck, 4> generateChecks(); + + /// Try to create add a new (pointer-difference, access size) pair to + /// DiffCheck for checking groups \p CGI and \p CGJ. If pointer-difference + /// checks cannot be used for the groups, set CanUseDiffCheck to false. + void tryToCreateDiffCheck(const RuntimeCheckingPtrGroup &CGI, + const RuntimeCheckingPtrGroup &CGJ); + + MemoryDepChecker &DC; /// Holds a pointer to the ScalarEvolution analysis. ScalarEvolution *SE; @@ -485,6 +526,13 @@ private: /// Set of run-time checks required to establish independence of /// otherwise may-aliasing pointers in the loop. SmallVector<RuntimePointerCheck, 4> Checks; + + /// Flag indicating if pointer-difference checks can be used + bool CanUseDiffCheck = true; + + /// A list of (pointer-difference, access size) pairs that can be used to + /// prove that there are no vectorization-preventing dependencies. + SmallVector<PointerDiffInfo> DiffChecks; }; /// Drive the analysis of memory accesses in the loop @@ -575,6 +623,11 @@ public: return HasDependenceInvolvingLoopInvariantAddress; } + /// Return the list of stores to invariant addresses. + const ArrayRef<StoreInst *> getStoresToInvariantAddresses() const { + return StoresToInvariantAddresses; + } + /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts /// them to a more usable form. All SCEV expressions during the analysis /// should be re-written (and therefore simplified) according to PSE. @@ -605,6 +658,11 @@ private: /// invariant. void collectStridedAccess(Value *LoadOrStoreInst); + // Emits the first unsafe memory dependence in a loop. + // Emits nothing if there are no unsafe dependences + // or if the dependences were not recorded. + void emitUnsafeDependenceRemark(); + std::unique_ptr<PredicatedScalarEvolution> PSE; /// We need to check that all of the pointers in this list are disjoint @@ -629,6 +687,9 @@ private: /// Indicator that there are non vectorizable stores to a uniform address. bool HasDependenceInvolvingLoopInvariantAddress = false; + /// List of stores to invariant addresses. + SmallVector<StoreInst *> StoresToInvariantAddresses; + /// The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. std::unique_ptr<OptimizationRemarkAnalysis> Report; diff --git a/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/llvm/include/llvm/Analysis/LoopAnalysisManager.h index d07e6977fed1..d22675a308aa 100644 --- a/llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ b/llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -29,7 +29,6 @@ #ifndef LLVM_ANALYSIS_LOOPANALYSISMANAGER_H #define LLVM_ANALYSIS_LOOPANALYSISMANAGER_H -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Analysis/LoopCacheAnalysis.h b/llvm/include/llvm/Analysis/LoopCacheAnalysis.h index 21882ebd0087..4c5083f3c980 100644 --- a/llvm/include/llvm/Analysis/LoopCacheAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopCacheAnalysis.h @@ -15,15 +15,17 @@ #define LLVM_ANALYSIS_LOOPCACHEANALYSIS_H #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/raw_ostream.h" namespace llvm { class AAResults; class DependenceInfo; +class Instruction; class LPMUpdater; +class raw_ostream; +class LoopInfo; +class Loop; class ScalarEvolution; class SCEV; class TargetTransformInfo; @@ -96,6 +98,10 @@ private: /// Attempt to delinearize the indexed reference. bool delinearize(const LoopInfo &LI); + /// Attempt to delinearize \p AccessFn for fixed-size arrays. + bool tryDelinearizeFixedSize(const SCEV *AccessFn, + SmallVectorImpl<const SCEV *> &Subscripts); + /// Return true if the index reference is invariant with respect to loop \p L. bool isLoopInvariant(const Loop &L) const; @@ -105,6 +111,13 @@ private: /// smaller than the cache line size \p CLS. bool isConsecutive(const Loop &L, unsigned CLS) const; + /// Retrieve the index of the subscript corresponding to the given loop \p + /// L. Return a zero-based positive index if the subscript index is + /// succesfully located and a negative value otherwise. For example given the + /// indexed reference 'A[i][2j+1][3k+2]', the call + /// 'getSubscriptIndex(loop-k)' would return value 2. + int getSubscriptIndex(const Loop &L) const; + /// Return the coefficient used in the rightmost dimension. const SCEV *getLastCoefficient() const; @@ -237,9 +250,10 @@ private: /// Sort the LoopCosts vector by decreasing cache cost. void sortLoopCosts() { - sort(LoopCosts, [](const LoopCacheCostTy &A, const LoopCacheCostTy &B) { - return A.second > B.second; - }); + stable_sort(LoopCosts, + [](const LoopCacheCostTy &A, const LoopCacheCostTy &B) { + return A.second > B.second; + }); } private: diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index a0ffdb07a7ec..9351b83ad747 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -44,7 +44,6 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" @@ -55,9 +54,10 @@ namespace llvm { class DominatorTree; +class InductionDescriptor; +class Instruction; class LoopInfo; class Loop; -class InductionDescriptor; class MDNode; class MemorySSAUpdater; class ScalarEvolution; @@ -112,6 +112,22 @@ public: /// parent is the innermost loop in which it is enclosed. LoopT *getParentLoop() const { return ParentLoop; } + /// Get the outermost loop in which this loop is contained. + /// This may be the loop itself, if it already is the outermost loop. + const LoopT *getOutermostLoop() const { + const LoopT *L = static_cast<const LoopT *>(this); + while (L->ParentLoop) + L = L->ParentLoop; + return L; + } + + LoopT *getOutermostLoop() { + LoopT *L = static_cast<LoopT *>(this); + while (L->ParentLoop) + L = L->ParentLoop; + return L; + } + /// This is a raw interface for bypassing addChildLoop. void setParentLoop(LoopT *L) { assert(!isInvalid() && "Loop not in a valid state!"); diff --git a/llvm/include/llvm/Analysis/LoopInfoImpl.h b/llvm/include/llvm/Analysis/LoopInfoImpl.h index b8b8330d0fe1..a96a698f3afb 100644 --- a/llvm/include/llvm/Analysis/LoopInfoImpl.h +++ b/llvm/include/llvm/Analysis/LoopInfoImpl.h @@ -14,7 +14,6 @@ #ifndef LLVM_ANALYSIS_LOOPINFOIMPL_H #define LLVM_ANALYSIS_LOOPINFOIMPL_H -#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" @@ -315,12 +314,11 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const { "Loop block has no in-loop predecessors!"); SmallVector<BlockT *, 2> OutsideLoopPreds; - std::for_each(GraphTraits<Inverse<BlockT *>>::child_begin(BB), - GraphTraits<Inverse<BlockT *>>::child_end(BB), - [&](BlockT *B) { - if (!contains(B)) - OutsideLoopPreds.push_back(B); - }); + for (BlockT *B : + llvm::make_range(GraphTraits<Inverse<BlockT *>>::child_begin(BB), + GraphTraits<Inverse<BlockT *>>::child_end(BB))) + if (!contains(B)) + OutsideLoopPreds.push_back(B); if (BB == getHeader()) { assert(!OutsideLoopPreds.empty() && "Loop is unreachable!"); @@ -455,8 +453,7 @@ static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT *> Backedges, InvBlockTraits::child_end(PredBB)); } else { // This is a discovered block. Find its outermost discovered loop. - while (LoopT *Parent = Subloop->getParentLoop()) - Subloop = Parent; + Subloop = Subloop->getOutermostLoop(); // If it is already discovered to be a subloop of this loop, continue. if (Subloop == L) diff --git a/llvm/include/llvm/Analysis/LoopPass.h b/llvm/include/llvm/Analysis/LoopPass.h index 0fd2a39eefc0..c5f08d0ae8af 100644 --- a/llvm/include/llvm/Analysis/LoopPass.h +++ b/llvm/include/llvm/Analysis/LoopPass.h @@ -14,13 +14,14 @@ #ifndef LLVM_ANALYSIS_LOOPPASS_H #define LLVM_ANALYSIS_LOOPPASS_H -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/LegacyPassManagers.h" #include "llvm/Pass.h" #include <deque> namespace llvm { +class Loop; +class LoopInfo; class LPPassManager; class Function; diff --git a/llvm/include/llvm/Analysis/LoopUnrollAnalyzer.h b/llvm/include/llvm/Analysis/LoopUnrollAnalyzer.h index 7cf8a081f9a2..eada6a647763 100644 --- a/llvm/include/llvm/Analysis/LoopUnrollAnalyzer.h +++ b/llvm/include/llvm/Analysis/LoopUnrollAnalyzer.h @@ -15,8 +15,9 @@ #ifndef LLVM_ANALYSIS_LOOPUNROLLANALYZER_H #define LLVM_ANALYSIS_LOOPUNROLLANALYZER_H -#include "llvm/Analysis/InstructionSimplify.h" -#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/InstVisitor.h" // This class is used to get an estimate of the optimization effects that we @@ -36,6 +37,8 @@ // And finally: // v = b[1] namespace llvm { +class Instruction; + class UnrolledInstAnalyzer : private InstVisitor<UnrolledInstAnalyzer, bool> { typedef InstVisitor<UnrolledInstAnalyzer, bool> Base; friend class InstVisitor<UnrolledInstAnalyzer, bool>; diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h index b1a81d5e7030..00e8d7d7dd4d 100644 --- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h @@ -9,6 +9,7 @@ #ifndef LLVM_ANALYSIS_MLINLINEADVISOR_H #define LLVM_ANALYSIS_MLINLINEADVISOR_H +#include "llvm/Analysis/FunctionPropertiesAnalysis.h" #include "llvm/Analysis/InlineAdvisor.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/MLModelRunner.h" @@ -19,6 +20,7 @@ #include <memory> namespace llvm { +class DiagnosticInfoOptimizationBase; class Module; class MLInlineAdvice; @@ -29,16 +31,19 @@ public: virtual ~MLInlineAdvisor() = default; - void onPassEntry() override; + void onPassEntry(LazyCallGraph::SCC *SCC) override; void onPassExit(LazyCallGraph::SCC *SCC) override; - int64_t getIRSize(const Function &F) const { return F.getInstructionCount(); } + int64_t getIRSize(Function &F) const { + return getCachedFPI(F).TotalInstructionCount; + } void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted); bool isForcedToStop() const { return ForceStop; } int64_t getLocalCalls(Function &F); const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); } + FunctionPropertiesInfo &getCachedFPI(Function &) const; protected: std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; @@ -60,11 +65,11 @@ protected: private: int64_t getModuleIRSize() const; + std::unique_ptr<InlineAdvice> + getSkipAdviceIfUnreachableCallsite(CallBase &CB); + void print(raw_ostream &OS) const override; - void print(raw_ostream &OS) const override { - OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount - << "\n"; - } + mutable DenseMap<const Function *, FunctionPropertiesInfo> FPICache; LazyCallGraph &CG; @@ -75,7 +80,7 @@ private: std::map<const LazyCallGraph::Node *, unsigned> FunctionLevels; const int32_t InitialIRSize = 0; int32_t CurrentIRSize = 0; - std::deque<const LazyCallGraph::Node *> NodesInLastSCC; + llvm::SmallPtrSet<const LazyCallGraph::Node *, 1> NodesInLastSCC; DenseSet<const LazyCallGraph::Node *> AllNodes; bool ForceStop = false; }; @@ -85,16 +90,7 @@ private: class MLInlineAdvice : public InlineAdvice { public: MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, - OptimizationRemarkEmitter &ORE, bool Recommendation) - : InlineAdvice(Advisor, CB, ORE, Recommendation), - CallerIRSize(Advisor->isForcedToStop() ? 0 - : Advisor->getIRSize(*Caller)), - CalleeIRSize(Advisor->isForcedToStop() ? 0 - : Advisor->getIRSize(*Callee)), - CallerAndCalleeEdges(Advisor->isForcedToStop() - ? 0 - : (Advisor->getLocalCalls(*Caller) + - Advisor->getLocalCalls(*Callee))) {} + OptimizationRemarkEmitter &ORE, bool Recommendation); virtual ~MLInlineAdvice() = default; void recordInliningImpl() override; @@ -108,13 +104,17 @@ public: const int64_t CallerIRSize; const int64_t CalleeIRSize; const int64_t CallerAndCalleeEdges; + void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const; private: void reportContextForRemark(DiagnosticInfoOptimizationBase &OR); - MLInlineAdvisor *getAdvisor() const { return static_cast<MLInlineAdvisor *>(Advisor); }; + // Make a copy of the FPI of the caller right before inlining. If inlining + // fails, we can just update the cache with that value. + const FunctionPropertiesInfo PreInlineCallerFPI; + Optional<FunctionPropertiesUpdater> FPU; }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/MLModelRunner.h b/llvm/include/llvm/Analysis/MLModelRunner.h index 669c02af0b3b..872c0e37f00e 100644 --- a/llvm/include/llvm/Analysis/MLModelRunner.h +++ b/llvm/include/llvm/Analysis/MLModelRunner.h @@ -10,10 +10,11 @@ #ifndef LLVM_ANALYSIS_MLMODELRUNNER_H #define LLVM_ANALYSIS_MLMODELRUNNER_H -#include "llvm/IR/LLVMContext.h" +#include "llvm/Analysis/TensorSpec.h" #include "llvm/IR/PassManager.h" namespace llvm { +class LLVMContext; /// MLModelRunner interface: abstraction of a mechanism for evaluating a /// tensorflow "saved model". @@ -41,7 +42,7 @@ public: getTensorUntyped(static_cast<size_t>(FeatureID))); } - virtual void *getTensorUntyped(size_t Index) = 0; + void *getTensorUntyped(size_t Index) { return InputBuffers[Index]; } const void *getTensorUntyped(size_t Index) const { return (const_cast<MLModelRunner *>(this))->getTensorUntyped(Index); } @@ -50,13 +51,27 @@ public: Kind getKind() const { return Type; } protected: - MLModelRunner(LLVMContext &Ctx, Kind Type) : Ctx(Ctx), Type(Type) { + MLModelRunner(LLVMContext &Ctx, Kind Type, size_t NrInputs) + : Ctx(Ctx), Type(Type), InputBuffers(NrInputs) { assert(Type != Kind::Unknown); } virtual void *evaluateUntyped() = 0; + void setUpBufferForTensor(size_t Index, const TensorSpec &Spec, + void *Buffer) { + if (!Buffer) { + OwnedBuffers.emplace_back(Spec.getTotalTensorBufferSize()); + Buffer = OwnedBuffers.back().data(); + } + InputBuffers[Index] = Buffer; + } + LLVMContext &Ctx; const Kind Type; + +private: + std::vector<void *> InputBuffers; + std::vector<std::vector<char *>> OwnedBuffers; }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index d5b60ee540e0..7ad83612880f 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -28,6 +28,7 @@ namespace llvm { class AllocaInst; +class AAResults; class Argument; class CallInst; class ConstantPointerNull; @@ -100,7 +101,10 @@ inline CallInst *isFreeCall(Value *I, const TargetLibraryInfo *TLI) { /// insertion or speculative execution of allocation routines. bool isAllocRemovable(const CallBase *V, const TargetLibraryInfo *TLI); -/// Gets the alignment argument for an aligned_alloc-like function +/// Gets the alignment argument for an aligned_alloc-like function, using either +/// built-in knowledge based on fuction names/signatures or allocalign +/// attributes. Note: the Value returned may not indicate a valid alignment, per +/// the definition of the allocalign attribute. Value *getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI); /// Return the size of the requested allocation. With a trivial mapper, this is @@ -111,12 +115,19 @@ Optional<APInt> getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, std::function<const Value*(const Value*)> Mapper); -/// If this allocation function initializes memory to a fixed value, return -/// said value in the requested type. Otherwise, return nullptr. -Constant *getInitialValueOfAllocation(const CallBase *Alloc, +/// If this is a call to an allocation function that initializes memory to a +/// fixed value, return said value in the requested type. Otherwise, return +/// nullptr. +Constant *getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty); +/// If a function is part of an allocation family (e.g. +/// malloc/realloc/calloc/free), return the identifier for its family +/// of functions. +Optional<StringRef> getAllocationFamily(const Value *I, + const TargetLibraryInfo *TLI); + //===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // @@ -143,6 +154,8 @@ struct ObjectSizeOpts { /// though they can't be evaluated. Otherwise, null is always considered to /// point to a 0 byte region of memory. bool NullIsUnknownSize = false; + /// If set, used for more accurate evaluation + AAResults *AA = nullptr; }; /// Compute the size of the object pointed by Ptr. Returns true and the @@ -162,8 +175,9 @@ bool getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, /// argument of the call to objectsize. Value *lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed); - - +Value *lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, + const TargetLibraryInfo *TLI, AAResults *AA, + bool MustSucceed); using SizeOffsetType = std::pair<APInt, APInt>; @@ -210,7 +224,6 @@ public: SizeOffsetType visitConstantPointerNull(ConstantPointerNull&); SizeOffsetType visitExtractElementInst(ExtractElementInst &I); SizeOffsetType visitExtractValueInst(ExtractValueInst &I); - SizeOffsetType visitGEPOperator(GEPOperator &GEP); SizeOffsetType visitGlobalAlias(GlobalAlias &GA); SizeOffsetType visitGlobalVariable(GlobalVariable &GV); SizeOffsetType visitIntToPtrInst(IntToPtrInst&); @@ -221,6 +234,12 @@ public: SizeOffsetType visitInstruction(Instruction &I); private: + SizeOffsetType findLoadSizeOffset( + LoadInst &LoadFrom, BasicBlock &BB, BasicBlock::iterator From, + SmallDenseMap<BasicBlock *, SizeOffsetType, 8> &VisitedBlocks, + unsigned &ScannedInstCount); + SizeOffsetType combineSizeOffset(SizeOffsetType LHS, SizeOffsetType RHS); + SizeOffsetType computeImpl(Value *V); bool CheckedZextOrTrunc(APInt &I); }; diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 23e50f601e04..dfac49445d75 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -36,6 +36,7 @@ class AnyMemTransferInst; class AnyMemIntrinsic; class TargetLibraryInfo; class VAArgInst; +class Value; // Represents the size of a MemoryLocation. Logically, it's an // Optional<uint63_t> that also carries a bit to represent whether the integer diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h index b41f5771bacd..8cadb6a4c912 100644 --- a/llvm/include/llvm/Analysis/MemorySSA.h +++ b/llvm/include/llvm/Analysis/MemorySSA.h @@ -66,6 +66,19 @@ /// MemoryDefs are not disambiguated because it would require multiple reaching /// definitions, which would require multiple phis, and multiple memoryaccesses /// per instruction. +/// +/// In addition to the def/use graph described above, MemoryDefs also contain +/// an "optimized" definition use. The "optimized" use points to some def +/// reachable through the memory def chain. The optimized def *may* (but is +/// not required to) alias the original MemoryDef, but no def *closer* to the +/// source def may alias it. As the name implies, the purpose of the optimized +/// use is to allow caching of clobber searches for memory defs. The optimized +/// def may be nullptr, in which case clients must walk the defining access +/// chain. +/// +/// When iterating the uses of a MemoryDef, both defining uses and optimized +/// uses will be encountered. If only one type is needed, the client must +/// filter the use walk. // //===----------------------------------------------------------------------===// @@ -73,30 +86,18 @@ #define LLVM_ANALYSIS_MEMORYSSA_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" -#include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/ADT/simple_ilist.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/PHITransAddr.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/DerivedUser.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" -#include "llvm/IR/Use.h" #include "llvm/IR/User.h" -#include "llvm/IR/Value.h" -#include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/CommandLine.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -106,11 +107,16 @@ namespace llvm { +template <class GraphType> struct GraphTraits; +class BasicBlock; class Function; class Instruction; +class LLVMContext; class MemoryAccess; class MemorySSAWalker; -class LLVMContext; +class Module; +class Use; +class Value; class raw_ostream; namespace MSSAHelpers { @@ -259,10 +265,11 @@ public: return MA->getValueID() == MemoryUseVal || MA->getValueID() == MemoryDefVal; } - // Sadly, these have to be public because they are needed in some of the - // iterators. + /// Do we have an optimized use? inline bool isOptimized() const; + /// Return the MemoryAccess associated with the optimized use, or nullptr. inline MemoryAccess *getOptimized() const; + /// Sets the optimized use for a MemoryDef. inline void setOptimized(MemoryAccess *); // Retrieve AliasResult type of the optimized access. Ideally this would be @@ -339,6 +346,9 @@ public: setOperand(0, DMA); } + /// Whether the MemoryUse is optimized. If ensureOptimizedUses() was called, + /// uses will usually be optimized, but this is not guaranteed (e.g. due to + /// invalidation and optimization limits.) bool isOptimized() const { return getDefiningAccess() && OptimizedID == getDefiningAccess()->getID(); } @@ -791,6 +801,13 @@ public: /// about the beginning or end of a block. enum InsertionPlace { Beginning, End, BeforeTerminator }; + /// By default, uses are *not* optimized during MemorySSA construction. + /// Calling this method will attempt to optimize all MemoryUses, if this has + /// not happened yet for this MemorySSA instance. This should be done if you + /// plan to query the clobbering access for most uses, or if you walk the + /// def-use chain of uses. + void ensureOptimizedUses(); + protected: // Used by Memory SSA dumpers and wrapper pass friend class MemorySSAPrinterLegacyPass; @@ -893,6 +910,7 @@ private: std::unique_ptr<CachingWalker<AliasAnalysis>> Walker; std::unique_ptr<SkipSelfWalker<AliasAnalysis>> SkipWalker; unsigned NextID = 0; + bool IsOptimized = false; }; /// Enables verification of MemorySSA. diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h index 3e5ebe9cb427..2bcd1a462871 100644 --- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h +++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h @@ -31,7 +31,6 @@ #ifndef LLVM_ANALYSIS_MEMORYSSAUPDATER_H #define LLVM_ANALYSIS_MEMORYSSAUPDATER_H -#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" @@ -39,7 +38,6 @@ #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" #include "llvm/Support/CFGDiff.h" -#include <utility> namespace llvm { @@ -47,6 +45,7 @@ class BasicBlock; class DominatorTree; class Instruction; class LoopBlocksRPO; +template <typename T, unsigned int N> class SmallSetVector; using ValueToValueMapTy = ValueMap<const Value *, WeakTrackingVH>; using PhiToDefMap = SmallDenseMap<MemoryPhi *, MemoryAccess *>; diff --git a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h index 071ccf96fe5b..72bd185b6c32 100644 --- a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h +++ b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h @@ -10,6 +10,7 @@ #ifndef LLVM_ANALYSIS_MODELUNDERTRAININGRUNNER_H #define LLVM_ANALYSIS_MODELUNDERTRAININGRUNNER_H +#include "llvm/Analysis/TensorSpec.h" #include "llvm/Config/llvm-config.h" #ifdef LLVM_HAVE_TF_API @@ -48,6 +49,11 @@ public: StringRef DecisionName, const std::vector<TensorSpec> &InputSpecs, StringRef OutputSpecsPathOverride = ""); + static std::unique_ptr<ModelUnderTrainingRunner> + createAndEnsureValid(LLVMContext &Ctx, const std::string &ModelPath, + StringRef DecisionName, + const std::vector<TensorSpec> &InputSpecs, + const std::vector<LoggedFeatureSpec> &OutputSpecs); private: ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath, @@ -58,7 +64,6 @@ private: const std::vector<LoggedFeatureSpec> OutputSpecs; Optional<TFModelEvaluator::EvaluationResult> LastEvaluationResult; void *evaluateUntyped() override; - void *getTensorUntyped(size_t Index) override; bool isValid() const { return !!Evaluator; } }; diff --git a/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h b/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h index 99aa315319b8..fa91e4f653d0 100644 --- a/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h +++ b/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h @@ -11,9 +11,9 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/raw_ostream.h" namespace llvm { +class raw_ostream; class ModuleDebugInfoPrinterPass : public PassInfoMixin<ModuleDebugInfoPrinterPass> { diff --git a/llvm/include/llvm/Analysis/MustExecute.h b/llvm/include/llvm/Analysis/MustExecute.h index 18a0bfee5730..1e4994207555 100644 --- a/llvm/include/llvm/Analysis/MustExecute.h +++ b/llvm/include/llvm/Analysis/MustExecute.h @@ -28,7 +28,6 @@ #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/raw_ostream.h" namespace llvm { @@ -42,6 +41,7 @@ class Instruction; class Loop; class LoopInfo; class PostDominatorTree; +class raw_ostream; /// Captures loop safety information. /// It keep information for loop blocks may throw exception or otherwise diff --git a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h index 5bcedf98865c..980b40500d7c 100644 --- a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h +++ b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h @@ -10,13 +10,9 @@ #ifndef LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H #define LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H -#include "llvm/Config/llvm-config.h" - -/// While not strictly necessary to conditionally compile this, it really -/// has no usecase outside the 'development' mode. -#ifdef LLVM_HAVE_TF_API #include "llvm/Analysis/MLModelRunner.h" -#include "llvm/Analysis/Utils/TFUtils.h" +#include "llvm/Analysis/TensorSpec.h" +#include "llvm/Config/llvm-config.h" namespace llvm { /// A pseudo model runner. We use it to store feature values when collecting /// logs for the default policy, in 'development' mode, but never ask it to @@ -34,10 +30,6 @@ private: void *evaluateUntyped() override { llvm_unreachable("We shouldn't call run on this model runner."); } - void *getTensorUntyped(size_t Index) override; - - std::vector<std::unique_ptr<char[]>> ValuesBuffer; }; } // namespace llvm -#endif // defined(LLVM_HAVE_TF_API) #endif // LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H diff --git a/llvm/include/llvm/Analysis/ObjCARCUtil.h b/llvm/include/llvm/Analysis/ObjCARCUtil.h index 385fa5422926..56faa20c4c6e 100644 --- a/llvm/include/llvm/Analysis/ObjCARCUtil.h +++ b/llvm/include/llvm/Analysis/ObjCARCUtil.h @@ -35,7 +35,7 @@ inline bool hasAttachedCallOpBundle(const CallBase *CB) { // functions. return !CB->getFunctionType()->getReturnType()->isVoidTy() && CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall) - .hasValue(); + .has_value(); } /// This function returns operand bundle clang_arc_attachedcall's argument, @@ -59,7 +59,7 @@ inline bool isRetainOrClaimRV(ARCInstKind Kind) { /// or UnsafeClaimRV. inline ARCInstKind getAttachedARCFunctionKind(const CallBase *CB) { Optional<Function *> Fn = getAttachedARCFunction(CB); - if (!Fn.hasValue()) + if (!Fn) return ARCInstKind::None; auto FnClass = GetFunctionClass(*Fn); assert(isRetainOrClaimRV(FnClass) && "unexpected ARC runtime function"); diff --git a/llvm/include/llvm/Analysis/OverflowInstAnalysis.h b/llvm/include/llvm/Analysis/OverflowInstAnalysis.h index 7523fb9392cd..761d20f17a8b 100644 --- a/llvm/include/llvm/Analysis/OverflowInstAnalysis.h +++ b/llvm/include/llvm/Analysis/OverflowInstAnalysis.h @@ -14,11 +14,9 @@ #ifndef LLVM_ANALYSIS_OVERFLOWINSTANALYSIS_H #define LLVM_ANALYSIS_OVERFLOWINSTANALYSIS_H -#include "llvm/IR/InstrTypes.h" - namespace llvm { -class Value; class Use; +class Value; /// Match one of the patterns up to the select/logic op: /// %Op0 = icmp ne i4 %X, 0 diff --git a/llvm/include/llvm/Analysis/PhiValues.h b/llvm/include/llvm/Analysis/PhiValues.h index c0e91c8b0bdf..ecbb8874b378 100644 --- a/llvm/include/llvm/Analysis/PhiValues.h +++ b/llvm/include/llvm/Analysis/PhiValues.h @@ -22,7 +22,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" diff --git a/llvm/include/llvm/Analysis/PostDominators.h b/llvm/include/llvm/Analysis/PostDominators.h index 296110d8d03b..4383113c8db1 100644 --- a/llvm/include/llvm/Analysis/PostDominators.h +++ b/llvm/include/llvm/Analysis/PostDominators.h @@ -102,10 +102,7 @@ template <> struct GraphTraits<PostDominatorTree*> } static nodes_iterator nodes_begin(PostDominatorTree *N) { - if (getEntryNode(N)) - return df_begin(getEntryNode(N)); - else - return df_end(getEntryNode(N)); + return df_begin(getEntryNode(N)); } static nodes_iterator nodes_end(PostDominatorTree *N) { diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h index 886800d8a0f5..773784ac418c 100644 --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -170,11 +170,11 @@ public: uint64_t getOrCompColdCountThreshold() const; /// Returns HotCountThreshold if set. uint64_t getHotCountThreshold() const { - return HotCountThreshold.getValueOr(0); + return HotCountThreshold.value_or(0); } /// Returns ColdCountThreshold if set. uint64_t getColdCountThreshold() const { - return ColdCountThreshold.getValueOr(0); + return ColdCountThreshold.value_or(0); } private: diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h index 78e9251da627..86206b2d5e9f 100644 --- a/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -26,22 +26,15 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstVisitor.h" -#include "llvm/IR/Instruction.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/Use.h" -#include "llvm/IR/User.h" -#include "llvm/Support/Casting.h" -#include <algorithm> #include <cassert> #include <type_traits> namespace llvm { +class DataLayout; +class Use; namespace detail { diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h index f93081d6f51d..612b977f1ffa 100644 --- a/llvm/include/llvm/Analysis/RegionInfo.h +++ b/llvm/include/llvm/Analysis/RegionInfo.h @@ -42,11 +42,9 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> #include <map> @@ -58,6 +56,7 @@ namespace llvm { +class BasicBlock; class DominanceFrontier; class Loop; class LoopInfo; @@ -67,6 +66,7 @@ template <class RegionTr> class RegionBase; class RegionInfo; template <class RegionTr> class RegionInfoBase; class RegionNode; +class raw_ostream; // Class to be specialized for different users of RegionInfo // (i.e. BasicBlocks or MachineBasicBlocks). This is only to avoid needing to @@ -242,7 +242,7 @@ public: /// /// You can obtain more examples by either calling /// -/// <tt> "opt -regions -analyze anyprogram.ll" </tt> +/// <tt> "opt -passes='print<regions>' anyprogram.ll" </tt> /// or /// <tt> "opt -view-regions-only anyprogram.ll" </tt> /// diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h index b694effb2229..561702db3790 100644 --- a/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -15,8 +15,6 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/iterator_range.h" -#include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/RegionInfo.h" @@ -24,7 +22,6 @@ #include "llvm/Config/llvm-config.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> #include <iterator> @@ -37,6 +34,7 @@ #define DEBUG_TYPE "region" namespace llvm { +class raw_ostream; //===----------------------------------------------------------------------===// /// RegionBase Implementation diff --git a/llvm/include/llvm/Analysis/RegionIterator.h b/llvm/include/llvm/Analysis/RegionIterator.h index fecb28725dcc..ba28b1b902ea 100644 --- a/llvm/include/llvm/Analysis/RegionIterator.h +++ b/llvm/include/llvm/Analysis/RegionIterator.h @@ -15,7 +15,6 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/Analysis/RegionInfo.h" -#include "llvm/IR/CFG.h" #include <cassert> #include <iterator> #include <type_traits> @@ -23,6 +22,7 @@ namespace llvm { class BasicBlock; +class RegionInfo; //===----------------------------------------------------------------------===// /// Hierarchical RegionNode successor iterator. diff --git a/llvm/include/llvm/Analysis/RegionPass.h b/llvm/include/llvm/Analysis/RegionPass.h index 5c7fa5f56693..dd5e6a1a3b24 100644 --- a/llvm/include/llvm/Analysis/RegionPass.h +++ b/llvm/include/llvm/Analysis/RegionPass.h @@ -15,7 +15,6 @@ #ifndef LLVM_ANALYSIS_REGIONPASS_H #define LLVM_ANALYSIS_REGIONPASS_H -#include "llvm/Analysis/RegionInfo.h" #include "llvm/IR/LegacyPassManagers.h" #include "llvm/Pass.h" #include <deque> @@ -23,6 +22,8 @@ namespace llvm { class Function; class RGPassManager; +class Region; +class RegionInfo; //===----------------------------------------------------------------------===// /// A pass that runs on each Region in a function. diff --git a/llvm/include/llvm/Analysis/RegionPrinter.h b/llvm/include/llvm/Analysis/RegionPrinter.h index 154ac35c486a..501a5406236e 100644 --- a/llvm/include/llvm/Analysis/RegionPrinter.h +++ b/llvm/include/llvm/Analysis/RegionPrinter.h @@ -14,6 +14,9 @@ #ifndef LLVM_ANALYSIS_REGIONPRINTER_H #define LLVM_ANALYSIS_REGIONPRINTER_H +#include "llvm/Analysis/DOTGraphTraitsPass.h" +#include "llvm/Analysis/RegionInfo.h" + namespace llvm { class FunctionPass; class Function; @@ -24,6 +27,13 @@ namespace llvm { FunctionPass *createRegionPrinterPass(); FunctionPass *createRegionOnlyPrinterPass(); + template <> + struct DOTGraphTraits<RegionNode *> : public DefaultDOTGraphTraits { + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} + + std::string getNodeLabel(RegionNode *Node, RegionNode *Graph); + }; + #ifndef NDEBUG /// Open a viewer to display the GraphViz vizualization of the analysis /// result. diff --git a/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h index 1bf2e853980c..bf1aaca2adbb 100644 --- a/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h +++ b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h @@ -15,11 +15,12 @@ #define LLVM_ANALYSIS_RELEASEMODEMODELRUNNER_H #include "llvm/Analysis/MLModelRunner.h" +#include "llvm/Analysis/TensorSpec.h" +#include "llvm/Support/ErrorHandling.h" #include <memory> #include <vector> -using namespace llvm; namespace llvm { /// ReleaseModeModelRunner - production mode implementation of the @@ -30,21 +31,20 @@ public: /// FeatureNames' type should be an indexed collection of std::string, like /// std::array or std::vector, that has a size() method. template <class FType> - ReleaseModeModelRunner(LLVMContext &Ctx, const FType &FeatureNames, + ReleaseModeModelRunner(LLVMContext &Ctx, const FType &InputSpec, StringRef DecisionName, StringRef FeedPrefix = "feed_", StringRef FetchPrefix = "fetch_") - : MLModelRunner(Ctx, MLModelRunner::Kind::Release), + : MLModelRunner(Ctx, MLModelRunner::Kind::Release, InputSpec.size()), CompiledModel(std::make_unique<TGen>()) { assert(CompiledModel && "The CompiledModel should be valid"); - const size_t FeatureCount = FeatureNames.size(); - FeatureIndices.resize(FeatureCount); - - for (size_t I = 0; I < FeatureCount; ++I) { + for (size_t I = 0; I < InputSpec.size(); ++I) { const int Index = - CompiledModel->LookupArgIndex(FeedPrefix.str() + FeatureNames[I]); - assert(Index >= 0 && "Cannot find Feature in inlining model"); - FeatureIndices[I] = Index; + CompiledModel->LookupArgIndex(FeedPrefix.str() + InputSpec[I].name()); + void *Buffer = nullptr; + if (Index >= 0) + Buffer = CompiledModel->arg_data(Index); + setUpBufferForTensor(I, InputSpec[I], Buffer); } ResultIndex = CompiledModel->LookupResultIndex(FetchPrefix.str() + @@ -64,15 +64,27 @@ private: return CompiledModel->result_data(ResultIndex); } - void *getTensorUntyped(size_t Index) override { - return reinterpret_cast<char *>( - CompiledModel->arg_data(FeatureIndices[Index])); - } - - std::vector<int32_t> FeatureIndices; int32_t ResultIndex = -1; std::unique_ptr<TGen> CompiledModel; }; + +/// A mock class satisfying the interface expected by ReleaseModeModelRunner for +/// its `TGen` parameter. Useful to avoid conditional compilation complexity, as +/// a compile-time replacement for a real AOT-ed model. +class NoopSavedModelImpl final { +#define NOOP_MODEL_ERRMSG \ + "The mock AOT-ed saved model is a compile-time stub and should not be " \ + "called." + +public: + NoopSavedModelImpl() = default; + int LookupArgIndex(const std::string &) { llvm_unreachable(NOOP_MODEL_ERRMSG); } + int LookupResultIndex(const std::string &) { llvm_unreachable(NOOP_MODEL_ERRMSG); } + void Run() { llvm_unreachable(NOOP_MODEL_ERRMSG); } + void *result_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } + void *arg_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } +#undef NOOP_MODEL_ERRMSG +}; } // namespace llvm #endif // LLVM_ANALYSIS_RELEASEMODEMODELRUNNER_H diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h index dc2efeafb568..0c5b566f60a4 100644 --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -11,11 +11,11 @@ #include "llvm/ADT/StringSet.h" #include "llvm/Analysis/InlineAdvisor.h" -#include "llvm/IR/LLVMContext.h" namespace llvm { class CallBase; class Function; +class LLVMContext; class Module; struct CallSiteFormat { @@ -53,10 +53,12 @@ struct ReplayInlinerSettings { /// Get call site location as a string with the given format std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format); -std::unique_ptr<InlineAdvisor> getReplayInlineAdvisor( - Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr<InlineAdvisor> OriginalAdvisor, - const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks); +std::unique_ptr<InlineAdvisor> +getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + LLVMContext &Context, + std::unique_ptr<InlineAdvisor> OriginalAdvisor, + const ReplayInlinerSettings &ReplaySettings, + bool EmitRemarks, InlineContext IC); /// Replay inline advisor that uses optimization remarks from inlining of /// previous build to guide current inlining. This is useful for inliner tuning. @@ -66,7 +68,7 @@ public: LLVMContext &Context, std::unique_ptr<InlineAdvisor> OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, - bool EmitRemarks); + bool EmitRemarks, InlineContext IC); std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; bool areReplayRemarksLoaded() const { return HasReplayRemarks; } diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index b16aa7017719..de1cc299f062 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -31,18 +31,12 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/ConstantRange.h" -#include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" #include "llvm/Pass.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Compiler.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <memory> @@ -50,12 +44,14 @@ namespace llvm { +class OverflowingBinaryOperator; class AssumptionCache; class BasicBlock; class Constant; class ConstantInt; class DataLayout; class DominatorTree; +class Function; class GEPOperator; class Instruction; class LLVMContext; @@ -71,6 +67,8 @@ class Type; class Value; enum SCEVTypes : unsigned short; +extern bool VerifySCEV; + /// This class represents an analyzed expression in the program. These are /// opaque objects that the client is not allowed to do much with directly. /// @@ -222,7 +220,7 @@ class SCEVPredicate : public FoldingSetNode { FoldingSetNodeIDRef FastID; public: - enum SCEVPredicateKind { P_Union, P_Equal, P_Wrap }; + enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap }; protected: SCEVPredicateKind Kind; @@ -249,10 +247,6 @@ public: /// Prints a textual representation of this predicate with an indentation of /// \p Depth. virtual void print(raw_ostream &OS, unsigned Depth = 0) const = 0; - - /// Returns the SCEV to which this predicate applies, or nullptr if this is - /// a SCEVUnionPredicate. - virtual const SCEV *getExpr() const = 0; }; inline raw_ostream &operator<<(raw_ostream &OS, const SCEVPredicate &P) { @@ -279,32 +273,35 @@ struct FoldingSetTrait<SCEVPredicate> : DefaultFoldingSetTrait<SCEVPredicate> { } }; -/// This class represents an assumption that two SCEV expressions are equal, -/// and this can be checked at run-time. -class SCEVEqualPredicate final : public SCEVPredicate { - /// We assume that LHS == RHS. +/// This class represents an assumption that the expression LHS Pred RHS +/// evaluates to true, and this can be checked at run-time. +class SCEVComparePredicate final : public SCEVPredicate { + /// We assume that LHS Pred RHS is true. + const ICmpInst::Predicate Pred; const SCEV *LHS; const SCEV *RHS; public: - SCEVEqualPredicate(const FoldingSetNodeIDRef ID, const SCEV *LHS, - const SCEV *RHS); + SCEVComparePredicate(const FoldingSetNodeIDRef ID, + const ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); /// Implementation of the SCEVPredicate interface bool implies(const SCEVPredicate *N) const override; void print(raw_ostream &OS, unsigned Depth = 0) const override; bool isAlwaysTrue() const override; - const SCEV *getExpr() const override; - /// Returns the left hand side of the equality. + ICmpInst::Predicate getPredicate() const { return Pred; } + + /// Returns the left hand side of the predicate. const SCEV *getLHS() const { return LHS; } - /// Returns the right hand side of the equality. + /// Returns the right hand side of the predicate. const SCEV *getRHS() const { return RHS; } /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const SCEVPredicate *P) { - return P->getKind() == P_Equal; + return P->getKind() == P_Compare; } }; @@ -396,7 +393,7 @@ public: IncrementWrapFlags getFlags() const { return Flags; } /// Implementation of the SCEVPredicate interface - const SCEV *getExpr() const override; + const SCEVAddRecExpr *getExpr() const; bool implies(const SCEVPredicate *N) const override; void print(raw_ostream &OS, unsigned Depth = 0) const override; bool isAlwaysTrue() const override; @@ -421,28 +418,20 @@ private: /// Vector with references to all predicates in this union. SmallVector<const SCEVPredicate *, 16> Preds; - /// Maps SCEVs to predicates for quick look-ups. - PredicateMap SCEVToPreds; + /// Adds a predicate to this union. + void add(const SCEVPredicate *N); public: - SCEVUnionPredicate(); + SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds); const SmallVectorImpl<const SCEVPredicate *> &getPredicates() const { return Preds; } - /// Adds a predicate to this union. - void add(const SCEVPredicate *N); - - /// Returns a reference to a vector containing all predicates which apply to - /// \p Expr. - ArrayRef<const SCEVPredicate *> getPredicatesForExpr(const SCEV *Expr); - /// Implementation of the SCEVPredicate interface bool isAlwaysTrue() const override; bool implies(const SCEVPredicate *N) const override; void print(raw_ostream &OS, unsigned Depth) const override; - const SCEV *getExpr() const override; /// We estimate the complexity of a union predicate as the size number of /// predicates in the union. @@ -556,6 +545,10 @@ public: /// Return true if the SCEV expression contains an undef value. bool containsUndefs(const SCEV *S) const; + /// Return true if the SCEV expression contains a Value that has been + /// optimised out and is now a nullptr. + bool containsErasedValue(const SCEV *S) const; + /// Return a SCEV expression for the full generality of the specified /// expression. const SCEV *getSCEV(Value *V); @@ -885,7 +878,7 @@ public: /// the answer to be correct. Predicates can be checked with run-time /// checks and can be used to perform loop versioning. const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Predicates); + SmallVector<const SCEVPredicate *, 4> &Predicates); /// When successful, this returns a SCEVConstant that is greater than or equal /// to (i.e. a "conservative over-approximation") of the value returend by @@ -1166,6 +1159,8 @@ public: } const SCEVPredicate *getEqualPredicate(const SCEV *LHS, const SCEV *RHS); + const SCEVPredicate *getComparePredicate(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); const SCEVPredicate * getWrapPredicate(const SCEVAddRecExpr *AR, @@ -1173,7 +1168,7 @@ public: /// Re-writes the SCEV according to the Predicates in \p A. const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L, - SCEVUnionPredicate &A); + const SCEVPredicate &A); /// Tries to convert the \p S expression to an AddRec expression, /// adding additional predicates to \p Preds as required. const SCEVAddRecExpr *convertSCEVToAddRecWithPredicates( @@ -1256,30 +1251,11 @@ private: HasRecMapType HasRecMap; /// The type for ExprValueMap. - using ValueOffsetPair = std::pair<Value *, ConstantInt *>; - using ValueOffsetPairSetVector = SmallSetVector<ValueOffsetPair, 4>; - using ExprValueMapType = DenseMap<const SCEV *, ValueOffsetPairSetVector>; + using ValueSetVector = SmallSetVector<Value *, 4>; + using ExprValueMapType = DenseMap<const SCEV *, ValueSetVector>; /// ExprValueMap -- This map records the original values from which /// the SCEV expr is generated from. - /// - /// We want to represent the mapping as SCEV -> ValueOffsetPair instead - /// of SCEV -> Value: - /// Suppose we know S1 expands to V1, and - /// S1 = S2 + C_a - /// S3 = S2 + C_b - /// where C_a and C_b are different SCEVConstants. Then we'd like to - /// expand S3 as V1 - C_a + C_b instead of expanding S2 literally. - /// It is helpful when S2 is a complex SCEV expr. - /// - /// In order to do that, we represent ExprValueMap as a mapping from - /// SCEV to ValueOffsetPair. We will save both S1->{V1, 0} and - /// S2->{V1, C_a} into the map when we create SCEV for V1. When S3 - /// is expanded, it will first expand S2 to V1 - C_a because of - /// S2->{V1, C_a} in the map, then expand S3 to V1 - C_a + C_b. - /// - /// Note: S->{V, Offset} in the ExprValueMap means S can be expanded - /// to V - Offset. ExprValueMapType ExprValueMap; /// The type for ValueExprMap. @@ -1310,7 +1286,7 @@ private: DenseMap<const SCEV *, uint32_t> MinTrailingZerosCache; /// Return the Value set from which the SCEV expr is generated. - ValueOffsetPairSetVector *getSCEVValues(const SCEV *S); + ArrayRef<Value *> getSCEVValues(const SCEV *S); /// Private helper method for the GetMinTrailingZeros method uint32_t GetMinTrailingZerosImpl(const SCEV *S); @@ -1369,17 +1345,17 @@ private: PoisoningVH<BasicBlock> ExitingBlock; const SCEV *ExactNotTaken; const SCEV *MaxNotTaken; - std::unique_ptr<SCEVUnionPredicate> Predicate; + SmallPtrSet<const SCEVPredicate *, 4> Predicates; explicit ExitNotTakenInfo(PoisoningVH<BasicBlock> ExitingBlock, const SCEV *ExactNotTaken, const SCEV *MaxNotTaken, - std::unique_ptr<SCEVUnionPredicate> Predicate) + const SmallPtrSet<const SCEVPredicate *, 4> &Predicates) : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), - MaxNotTaken(ExactNotTaken), Predicate(std::move(Predicate)) {} + MaxNotTaken(ExactNotTaken), Predicates(Predicates) {} bool hasAlwaysTruePredicate() const { - return !Predicate || Predicate->isAlwaysTrue(); + return Predicates.empty(); } }; @@ -1452,7 +1428,7 @@ private: /// vector, this information can contain them and therefore a /// SCEVPredicate argument should be added to getExact. const SCEV *getExact(const Loop *L, ScalarEvolution *SE, - SCEVUnionPredicate *Predicates = nullptr) const; + SmallVector<const SCEVPredicate *, 4> *Predicates = nullptr) const; /// Return the number of times this loop exit may fall through to the back /// edge, or SCEVCouldNotCompute. The loop is guaranteed not to exit via @@ -1599,9 +1575,17 @@ private: ConstantRange getRangeForUnknownRecurrence(const SCEVUnknown *U); /// We know that there is no SCEV for the specified value. Analyze the - /// expression. + /// expression recursively. const SCEV *createSCEV(Value *V); + /// We know that there is no SCEV for the specified value. Create a new SCEV + /// for \p V iteratively. + const SCEV *createSCEVIter(Value *V); + /// Collect operands of \p V for which SCEV expressions should be constructed + /// first. Returns a SCEV directly if it can be constructed trivially for \p + /// V. + const SCEV *getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops); + /// Provide the special handling we need to analyze PHI SCEVs. const SCEV *createNodeForPHI(PHINode *PN); @@ -1619,8 +1603,22 @@ private: /// is either a select instruction or a phi node). \p I is the instruction /// being processed, and it is assumed equivalent to "Cond ? TrueVal : /// FalseVal". - const SCEV *createNodeForSelectOrPHI(Instruction *I, Value *Cond, - Value *TrueVal, Value *FalseVal); + const SCEV *createNodeForSelectOrPHIInstWithICmpInstCond(Instruction *I, + ICmpInst *Cond, + Value *TrueVal, + Value *FalseVal); + + /// See if we can model this select-like instruction via umin_seq expression. + const SCEV *createNodeForSelectOrPHIViaUMinSeq(Value *I, Value *Cond, + Value *TrueVal, + Value *FalseVal); + + /// Given a value \p V, which is a select-like instruction (currently this is + /// either a select instruction or a phi node), which is assumed equivalent to + /// Cond ? TrueVal : FalseVal + /// see if we can model it as a SCEV expression. + const SCEV *createNodeForSelectOrPHI(Value *V, Value *Cond, Value *TrueVal, + Value *FalseVal); /// Provide the special handling we need to analyze GEP SCEVs. const SCEV *createNodeForGEP(GEPOperator *GEP); @@ -2097,6 +2095,11 @@ private: /// `UniqueSCEVs`. Return if found, else nullptr. SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef<const SCEV *> Ops); + /// Get reachable blocks in this function, making limited use of SCEV + /// reasoning about conditions. + void getReachableBlocks(SmallPtrSetImpl<BasicBlock *> &Reachable, + Function &F); + FoldingSet<SCEV> UniqueSCEVs; FoldingSet<SCEVPredicate> UniquePreds; BumpPtrAllocator SCEVAllocator; @@ -2182,7 +2185,7 @@ class PredicatedScalarEvolution { public: PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L); - const SCEVUnionPredicate &getUnionPredicate() const; + const SCEVPredicate &getPredicate() const; /// Returns the SCEV expression of V, in the context of the current SCEV /// predicate. The order of transformations applied on the expression of V @@ -2251,7 +2254,7 @@ private: /// The SCEVPredicate that forms our context. We will rewrite all /// expressions assuming that this predicate true. - SCEVUnionPredicate Preds; + std::unique_ptr<SCEVUnionPredicate> Preds; /// Marks the version of the SCEV predicate used. When rewriting a SCEV /// expression we mark it with the version of the predicate. We use this to diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h b/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h index ebd427354cee..15e27283021c 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h @@ -14,13 +14,14 @@ #define LLVM_ANALYSIS_SCALAREVOLUTIONALIASANALYSIS_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/ScalarEvolutionExpressions.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" #include "llvm/Pass.h" namespace llvm { +class Function; +class ScalarEvolution; +class SCEV; + /// A simple alias analysis implementation that uses ScalarEvolution to answer /// queries. class SCEVAAResult : public AAResultBase<SCEVAAResult> { diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index cd8e5fab6766..b29854cddc66 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -14,13 +14,11 @@ #define LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -31,9 +29,11 @@ namespace llvm { class APInt; class Constant; +class ConstantInt; class ConstantRange; class Loop; class Type; +class Value; enum SCEVTypes : unsigned short { // These should be ordered in terms of increasing complexity to make the @@ -699,8 +699,11 @@ public: case scUMinExpr: case scSequentialUMinExpr: case scAddRecExpr: - for (const auto *Op : cast<SCEVNAryExpr>(S)->operands()) + for (const auto *Op : cast<SCEVNAryExpr>(S)->operands()) { push(Op); + if (Visitor.isDone()) + break; + } continue; case scUDivExpr: { const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionNormalization.h b/llvm/include/llvm/Analysis/ScalarEvolutionNormalization.h index 6ab92a3a977f..da420ff1e6d2 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionNormalization.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionNormalization.h @@ -35,7 +35,7 @@ #ifndef LLVM_ANALYSIS_SCALAREVOLUTIONNORMALIZATION_H #define LLVM_ANALYSIS_SCALAREVOLUTIONNORMALIZATION_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallPtrSet.h" namespace llvm { diff --git a/llvm/include/llvm/Analysis/ScalarFuncs.def b/llvm/include/llvm/Analysis/ScalarFuncs.def new file mode 100644 index 000000000000..2ed9be538091 --- /dev/null +++ b/llvm/include/llvm/Analysis/ScalarFuncs.def @@ -0,0 +1,117 @@ +//===-- ScalarFuncs.def - Library information ----------*- C++ -*----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// This .def file creates mapping from standard IEEE math functions +// their corresponding entries in the IBM MASS (scalar) library. +// LLVM intrinsic math functions will be handled in PPCISelLowing to +// allow existing optimizations like pow(x,0.5) --> sqrt(x). + +#if defined(TLI_DEFINE_SCALAR_MASS_FUNCS) +#define TLI_DEFINE_SCALAR_MASS_FUNC(SCAL, MASSENTRY) {SCAL, MASSENTRY}, +#endif + +TLI_DEFINE_SCALAR_MASS_FUNC("acosf", "__xl_acosf") +TLI_DEFINE_SCALAR_MASS_FUNC("__acosf_finite", "__xl_acosf") +TLI_DEFINE_SCALAR_MASS_FUNC("acos", "__xl_acos") +TLI_DEFINE_SCALAR_MASS_FUNC("__acos_finite", "__xl_acos") + +TLI_DEFINE_SCALAR_MASS_FUNC("acoshf", "__xl_acoshf") +TLI_DEFINE_SCALAR_MASS_FUNC("__acoshf_finite", "__xl_acoshf") +TLI_DEFINE_SCALAR_MASS_FUNC("acosh", "__xl_acosh") +TLI_DEFINE_SCALAR_MASS_FUNC("__acosh_finite", "__xl_acosh") + +TLI_DEFINE_SCALAR_MASS_FUNC("asinf", "__xl_asinf") +TLI_DEFINE_SCALAR_MASS_FUNC("__asinf_finite", "__xl_asinf") +TLI_DEFINE_SCALAR_MASS_FUNC("asin", "__xl_asin") +TLI_DEFINE_SCALAR_MASS_FUNC("__asin_finite", "__xl_asin") + +TLI_DEFINE_SCALAR_MASS_FUNC("asinhf", "__xl_asinhf") +TLI_DEFINE_SCALAR_MASS_FUNC("asinh", "__xl_asinh") + +TLI_DEFINE_SCALAR_MASS_FUNC("atanf", "__xl_atanf") +TLI_DEFINE_SCALAR_MASS_FUNC("atan", "__xl_atan") + +TLI_DEFINE_SCALAR_MASS_FUNC("atan2f", "__xl_atan2f") +TLI_DEFINE_SCALAR_MASS_FUNC("__atan2f_finite", "__xl_atan2f") +TLI_DEFINE_SCALAR_MASS_FUNC("atan2", "__xl_atan2") +TLI_DEFINE_SCALAR_MASS_FUNC("__atan2_finite", "__xl_atan2") + +TLI_DEFINE_SCALAR_MASS_FUNC("atanhf", "__xl_atanhf") +TLI_DEFINE_SCALAR_MASS_FUNC("__atanhf_finite", "__xl_atanhf") +TLI_DEFINE_SCALAR_MASS_FUNC("atanh", "__xl_atanh") +TLI_DEFINE_SCALAR_MASS_FUNC("__atanh_finite", "__xl_atanh") + +TLI_DEFINE_SCALAR_MASS_FUNC("cbrtf", "__xl_cbrtf") +TLI_DEFINE_SCALAR_MASS_FUNC("cbrt", "__xl_cbrt") + +TLI_DEFINE_SCALAR_MASS_FUNC("cosf", "__xl_cosf") +TLI_DEFINE_SCALAR_MASS_FUNC("cos", "__xl_cos") + +TLI_DEFINE_SCALAR_MASS_FUNC("coshf", "__xl_coshf") +TLI_DEFINE_SCALAR_MASS_FUNC("__coshf_finite", "__xl_coshf") +TLI_DEFINE_SCALAR_MASS_FUNC("cosh", "__xl_cosh") +TLI_DEFINE_SCALAR_MASS_FUNC("__cosh_finite", "__xl_cosh") + +TLI_DEFINE_SCALAR_MASS_FUNC("erff", "__xl_erff") +TLI_DEFINE_SCALAR_MASS_FUNC("erf", "__xl_erf") + +TLI_DEFINE_SCALAR_MASS_FUNC("erfcf", "__xl_erfcf") +TLI_DEFINE_SCALAR_MASS_FUNC("erfc", "__xl_erfc") + +TLI_DEFINE_SCALAR_MASS_FUNC("expf", "__xl_expf") +TLI_DEFINE_SCALAR_MASS_FUNC("__expf_finite", "__xl_expf") +TLI_DEFINE_SCALAR_MASS_FUNC("exp", "__xl_exp") +TLI_DEFINE_SCALAR_MASS_FUNC("__exp_finite", "__xl_exp") + +TLI_DEFINE_SCALAR_MASS_FUNC("expm1f", "__xl_expm1f") +TLI_DEFINE_SCALAR_MASS_FUNC("expm1", "__xl_expm1") + +TLI_DEFINE_SCALAR_MASS_FUNC("hypotf", "__xl_hypotf") +TLI_DEFINE_SCALAR_MASS_FUNC("hypot", "__xl_hypot") + +TLI_DEFINE_SCALAR_MASS_FUNC("lgammaf", "__xl_lgammaf") +TLI_DEFINE_SCALAR_MASS_FUNC("lgamma", "__xl_lgamma") + +TLI_DEFINE_SCALAR_MASS_FUNC("logf", "__xl_logf") +TLI_DEFINE_SCALAR_MASS_FUNC("__logf_finite", "__xl_logf") +TLI_DEFINE_SCALAR_MASS_FUNC("log", "__xl_log") +TLI_DEFINE_SCALAR_MASS_FUNC("__log_finite", "__xl_log") + +TLI_DEFINE_SCALAR_MASS_FUNC("log10f", "__xl_log10f") +TLI_DEFINE_SCALAR_MASS_FUNC("__log10f_finite", "__xl_log10f") +TLI_DEFINE_SCALAR_MASS_FUNC("log10", "__xl_log10") +TLI_DEFINE_SCALAR_MASS_FUNC("__log10_finite", "__xl_log10") + +TLI_DEFINE_SCALAR_MASS_FUNC("log1pf", "__xl_log1pf") +TLI_DEFINE_SCALAR_MASS_FUNC("log1p", "__xl_log1p") + +TLI_DEFINE_SCALAR_MASS_FUNC("powf", "__xl_powf") +TLI_DEFINE_SCALAR_MASS_FUNC("__powf_finite", "__xl_powf") +TLI_DEFINE_SCALAR_MASS_FUNC("pow", "__xl_pow") +TLI_DEFINE_SCALAR_MASS_FUNC("__pow_finite", "__xl_pow") + +TLI_DEFINE_SCALAR_MASS_FUNC("rsqrt", "__xl_rsqrt") + +TLI_DEFINE_SCALAR_MASS_FUNC("sinf", "__xl_sinf") +TLI_DEFINE_SCALAR_MASS_FUNC("sin", "__xl_sin") + +TLI_DEFINE_SCALAR_MASS_FUNC("sinhf", "__xl_sinhf") +TLI_DEFINE_SCALAR_MASS_FUNC("__sinhf_finite", "__xl_sinhf") +TLI_DEFINE_SCALAR_MASS_FUNC("sinh", "__xl_sinh") +TLI_DEFINE_SCALAR_MASS_FUNC("__sinh_finite", "__xl_sinh") + +TLI_DEFINE_SCALAR_MASS_FUNC("sqrt", "__xl_sqrt") + +TLI_DEFINE_SCALAR_MASS_FUNC("tanf", "__xl_tanf") +TLI_DEFINE_SCALAR_MASS_FUNC("tan", "__xl_tan") + +TLI_DEFINE_SCALAR_MASS_FUNC("tanhf", "__xl_tanhf") +TLI_DEFINE_SCALAR_MASS_FUNC("tanh", "__xl_tanh") + +#undef TLI_DEFINE_SCALAR_MASS_FUNCS +#undef TLI_DEFINE_SCALAR_MASS_FUNC diff --git a/llvm/include/llvm/Analysis/SparsePropagation.h b/llvm/include/llvm/Analysis/SparsePropagation.h index 6eb6d5518a41..428238c5fa0b 100644 --- a/llvm/include/llvm/Analysis/SparsePropagation.h +++ b/llvm/include/llvm/Analysis/SparsePropagation.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_SPARSEPROPAGATION_H #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/Debug.h" #include <set> diff --git a/llvm/include/llvm/Analysis/StackLifetime.h b/llvm/include/llvm/Analysis/StackLifetime.h index 239aec4e258b..7fd88362276a 100644 --- a/llvm/include/llvm/Analysis/StackLifetime.h +++ b/llvm/include/llvm/Analysis/StackLifetime.h @@ -14,10 +14,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/raw_ostream.h" -#include <cassert> #include <utility> namespace llvm { @@ -26,6 +24,7 @@ class AllocaInst; class BasicBlock; class Function; class Instruction; +class IntrinsicInst; /// Compute live ranges of allocas. /// Live ranges are represented as sets of "interesting" instructions, which are diff --git a/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h b/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h index cfc1e20255d1..e6e3efbe0fcb 100644 --- a/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h @@ -16,18 +16,18 @@ #ifndef LLVM_ANALYSIS_SYNCDEPENDENCEANALYSIS_H #define LLVM_ANALYSIS_SYNCDEPENDENCEANALYSIS_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/Analysis/LoopInfo.h" #include <map> #include <memory> #include <unordered_map> +#include <vector> namespace llvm { class BasicBlock; class DominatorTree; +class Instruction; +class LoopInfo; class PostDominatorTree; using ConstBlockSet = SmallPtrSet<const BasicBlock *, 4>; diff --git a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h index f9bac739cee6..458b599f2937 100644 --- a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h +++ b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h @@ -13,7 +13,7 @@ #ifndef LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H #define LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Support/ScaledNumber.h" diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h index 1df0530e40e6..3a7218b10b97 100644 --- a/llvm/include/llvm/Analysis/TargetFolder.h +++ b/llvm/include/llvm/Analysis/TargetFolder.h @@ -21,12 +21,14 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/InstrTypes.h" #include "llvm/IR/IRBuilderFolder.h" +#include "llvm/IR/Operator.h" namespace llvm { +class Constant; class DataLayout; +class Type; /// TargetFolder - Create constants with target dependent folding. class TargetFolder final : public IRBuilderFolder { @@ -48,31 +50,45 @@ public: // Return an existing value or a constant if the operation can be simplified. // Otherwise return nullptr. //===--------------------------------------------------------------------===// - Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, - bool HasNSW = false) const override { + + Value *FoldBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); if (LC && RC) - return Fold(ConstantExpr::getAdd(LC, RC, HasNUW, HasNSW)); + return Fold(ConstantExpr::get(Opc, LC, RC)); return nullptr; } - Value *FoldAnd(Value *LHS, Value *RHS) const override { + Value *FoldExactBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool IsExact) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); if (LC && RC) - return Fold(ConstantExpr::getAnd(LC, RC)); + return Fold(ConstantExpr::get( + Opc, LC, RC, IsExact ? PossiblyExactOperator::IsExact : 0)); return nullptr; } - Value *FoldOr(Value *LHS, Value *RHS) const override { + Value *FoldNoWrapBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool HasNUW, bool HasNSW) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); - if (LC && RC) - return Fold(ConstantExpr::getOr(LC, RC)); + if (LC && RC) { + unsigned Flags = 0; + if (HasNUW) + Flags |= OverflowingBinaryOperator::NoUnsignedWrap; + if (HasNSW) + Flags |= OverflowingBinaryOperator::NoSignedWrap; + return Fold(ConstantExpr::get(Opc, LC, RC, Flags)); + } return nullptr; } + Value *FoldBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + FastMathFlags FMF) const override { + return FoldBinOp(Opc, LHS, RHS); + } Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); @@ -105,82 +121,56 @@ public: return nullptr; } - //===--------------------------------------------------------------------===// - // Binary Operators - //===--------------------------------------------------------------------===// - - Constant *CreateFAdd(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getFAdd(LHS, RHS)); - } - Constant *CreateSub(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return Fold(ConstantExpr::getSub(LHS, RHS, HasNUW, HasNSW)); - } - Constant *CreateFSub(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getFSub(LHS, RHS)); - } - Constant *CreateMul(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return Fold(ConstantExpr::getMul(LHS, RHS, HasNUW, HasNSW)); - } - Constant *CreateFMul(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getFMul(LHS, RHS)); - } - Constant *CreateUDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return Fold(ConstantExpr::getUDiv(LHS, RHS, isExact)); - } - Constant *CreateSDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return Fold(ConstantExpr::getSDiv(LHS, RHS, isExact)); - } - Constant *CreateFDiv(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getFDiv(LHS, RHS)); - } - Constant *CreateURem(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getURem(LHS, RHS)); - } - Constant *CreateSRem(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getSRem(LHS, RHS)); - } - Constant *CreateFRem(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getFRem(LHS, RHS)); - } - Constant *CreateShl(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return Fold(ConstantExpr::getShl(LHS, RHS, HasNUW, HasNSW)); - } - Constant *CreateLShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return Fold(ConstantExpr::getLShr(LHS, RHS, isExact)); + Value *FoldExtractValue(Value *Agg, + ArrayRef<unsigned> IdxList) const override { + if (auto *CAgg = dyn_cast<Constant>(Agg)) + return ConstantFoldExtractValueInstruction(CAgg, IdxList); + return nullptr; + }; + + Value *FoldInsertValue(Value *Agg, Value *Val, + ArrayRef<unsigned> IdxList) const override { + auto *CAgg = dyn_cast<Constant>(Agg); + auto *CVal = dyn_cast<Constant>(Val); + if (CAgg && CVal) + return ConstantFoldInsertValueInstruction(CAgg, CVal, IdxList); + return nullptr; } - Constant *CreateAShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return Fold(ConstantExpr::getAShr(LHS, RHS, isExact)); + + Value *FoldExtractElement(Value *Vec, Value *Idx) const override { + auto *CVec = dyn_cast<Constant>(Vec); + auto *CIdx = dyn_cast<Constant>(Idx); + if (CVec && CIdx) + return Fold(ConstantExpr::getExtractElement(CVec, CIdx)); + return nullptr; } - Constant *CreateXor(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getXor(LHS, RHS)); + + Value *FoldInsertElement(Value *Vec, Value *NewElt, + Value *Idx) const override { + auto *CVec = dyn_cast<Constant>(Vec); + auto *CNewElt = dyn_cast<Constant>(NewElt); + auto *CIdx = dyn_cast<Constant>(Idx); + if (CVec && CNewElt && CIdx) + return Fold(ConstantExpr::getInsertElement(CVec, CNewElt, CIdx)); + return nullptr; } - Constant *CreateBinOp(Instruction::BinaryOps Opc, - Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::get(Opc, LHS, RHS)); + Value *FoldShuffleVector(Value *V1, Value *V2, + ArrayRef<int> Mask) const override { + auto *C1 = dyn_cast<Constant>(V1); + auto *C2 = dyn_cast<Constant>(V2); + if (C1 && C2) + return Fold(ConstantExpr::getShuffleVector(C1, C2, Mask)); + return nullptr; } //===--------------------------------------------------------------------===// // Unary Operators //===--------------------------------------------------------------------===// - Constant *CreateNeg(Constant *C, - bool HasNUW = false, bool HasNSW = false) const override { - return Fold(ConstantExpr::getNeg(C, HasNUW, HasNSW)); - } Constant *CreateFNeg(Constant *C) const override { return Fold(ConstantExpr::getFNeg(C)); } - Constant *CreateNot(Constant *C) const override { - return Fold(ConstantExpr::getNot(C)); - } Constant *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const override { return Fold(ConstantExpr::get(Opc, C)); @@ -252,34 +242,6 @@ public: Constant *RHS) const override { return Fold(ConstantExpr::getCompare(P, LHS, RHS)); } - - //===--------------------------------------------------------------------===// - // Other Instructions - //===--------------------------------------------------------------------===// - - Constant *CreateExtractElement(Constant *Vec, Constant *Idx) const override { - return Fold(ConstantExpr::getExtractElement(Vec, Idx)); - } - - Constant *CreateInsertElement(Constant *Vec, Constant *NewElt, - Constant *Idx) const override { - return Fold(ConstantExpr::getInsertElement(Vec, NewElt, Idx)); - } - - Constant *CreateShuffleVector(Constant *V1, Constant *V2, - ArrayRef<int> Mask) const override { - return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask)); - } - - Constant *CreateExtractValue(Constant *Agg, - ArrayRef<unsigned> IdxList) const override { - return Fold(ConstantExpr::getExtractValue(Agg, IdxList)); - } - - Constant *CreateInsertValue(Constant *Agg, Constant *Val, - ArrayRef<unsigned> IdxList) const override { - return Fold(ConstantExpr::getInsertValue(Agg, Val, IdxList)); - } }; } diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index 17d1e3f770c1..7bfda0124de7 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -12,14 +12,15 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" -#include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace llvm { + template <typename T> class ArrayRef; +class Function; +class Module; class Triple; /// Describes a possible vectorization of a function. @@ -49,7 +50,7 @@ class TargetLibraryInfoImpl { friend class TargetLibraryInfo; unsigned char AvailableArray[(NumLibFuncs+3)/4]; - llvm::DenseMap<unsigned, std::string> CustomNames; + DenseMap<unsigned, std::string> CustomNames; static StringLiteral const StandardNames[NumLibFuncs]; bool ShouldExtI32Param, ShouldExtI32Return, ShouldSignExtI32Param; unsigned SizeOfInt; @@ -279,6 +280,13 @@ public: return B == OverrideAsUnavailable; } + /// Return true if the function type FTy is valid for the library function + /// F, regardless of whether the function is available. + bool isValidProtoForLibFunc(const FunctionType &FTy, LibFunc F, + const Module &M) const { + return Impl->isValidProtoForLibFunc(FTy, F, M); + } + /// Searches for a particular function name. /// /// If it is one of the known library functions, return true and set F to the diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 7412e050322e..372f17cfc7ff 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -21,13 +21,13 @@ #ifndef LLVM_ANALYSIS_TARGETTRANSFORMINFO_H #define LLVM_ANALYSIS_TARGETTRANSFORMINFO_H +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/IR/FMF.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/BranchProbability.h" -#include "llvm/Support/DataTypes.h" #include "llvm/Support/InstructionCost.h" #include <functional> #include <utility> @@ -617,8 +617,8 @@ public: Instruction *I = nullptr) const; /// Return true if LSR cost of C1 is lower than C1. - bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, - TargetTransformInfo::LSRCost &C2) const; + bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2) const; /// Return true if LSR major cost is number of registers. Targets which /// implement their own isLSRCostLess and unset number of registers as major @@ -659,6 +659,10 @@ public: /// Return true if the target supports nontemporal load. bool isLegalNTLoad(Type *DataType, Align Alignment) const; + /// \Returns true if the target supports broadcasting a load to a vector of + /// type <NumElements x ElementTy>. + bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const; + /// Return true if the target supports masked scatter. bool isLegalMaskedScatter(Type *DataType, Align Alignment) const; /// Return true if the target supports masked gather. @@ -675,6 +679,16 @@ public: /// Return true if the target supports masked expand load. bool isLegalMaskedExpandLoad(Type *DataType) const; + /// Return true if this is an alternating opcode pattern that can be lowered + /// to a single instruction on the target. In X86 this is for the addsub + /// instruction which corrsponds to a Shuffle + Fadd + FSub pattern in IR. + /// This function expectes two opcodes: \p Opcode1 and \p Opcode2 being + /// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0` + /// when \p Opcode0 is selected and `1` when Opcode1 is selected. + /// \p VecTy is the vector type of the instruction to be generated. + bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, + const SmallBitVector &OpcodeMask) const; + /// Return true if we should be enabling ordered reductions for the target. bool enableOrderedReductions() const; @@ -727,7 +741,7 @@ public: bool isTypeLegal(Type *Ty) const; /// Returns the estimated number of registers required to represent \p Ty. - InstructionCost getRegUsageForType(Type *Ty) const; + unsigned getRegUsageForType(Type *Ty) const; /// Return true if switches should be turned into lookup tables for the /// target. @@ -762,6 +776,9 @@ public: /// the scalarization cost of a load/store. bool supportsEfficientVectorElementLoadStore() const; + /// If the target supports tail calls. + bool supportsTailCalls() const; + /// Don't restrict interleaved unrolling to small loops. bool enableAggressiveInterleaving(bool LoopHasReductions) const; @@ -934,7 +951,8 @@ public: /// creating vectors that span multiple vector registers. /// If false, the vectorization factor will be chosen based on the /// size of the widest element type. - bool shouldMaximizeVectorBandwidth() const; + /// \p K Register Kind for vectorization. + bool shouldMaximizeVectorBandwidth(TargetTransformInfo::RegisterKind K) const; /// \return The minimum vectorization factor for types of given element /// bit width, or 0 if there is no minimum VF. The returned value only @@ -947,6 +965,17 @@ public: /// Currently only used by the SLP vectorizer. unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const; + /// \return The minimum vectorization factor for the store instruction. Given + /// the initial estimation of the minimum vector factor and store value type, + /// it tries to find possible lowest VF, which still might be profitable for + /// the vectorization. + /// \param VF Initial estimation of the minimum vector factor. + /// \param ScalarMemTy Scalar memory type of the store operation. + /// \param ScalarValTy Scalar type of the stored value. + /// Currently only used by the SLP vectorizer. + unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, + Type *ScalarValTy) const; + /// \return True if it should be considered for address type promotion. /// \p AllowPromotionWithoutCommonHeader Set true if promoting \p I is /// profitable without finding other extensions fed by the same input. @@ -1045,11 +1074,14 @@ public: /// The exact mask may be passed as Mask, or else the array will be empty. /// The index and subtype parameters are used by the subvector insertion and /// extraction shuffle kinds to show the insert/extract point and the type of - /// the subvector being inserted/extracted. + /// the subvector being inserted/extracted. The operands of the shuffle can be + /// passed through \p Args, which helps improve the cost estimation in some + /// cases, like in broadcast loads. /// NOTE: For subvector extractions Tp represents the source type. InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask = None, int Index = 0, - VectorType *SubTp = nullptr) const; + VectorType *SubTp = nullptr, + ArrayRef<const Value *> Args = None) const; /// Represents a hint about the context in which a cast is used. /// @@ -1283,9 +1315,11 @@ public: Type *ExpectedType) const; /// \returns The type to use in a loop expansion of a memcpy call. - Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, - unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const; + Type * + getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, unsigned DestAddrSpace, + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicElementSize = None) const; /// \param[out] OpsOut The operand types to copy RemainingBytes of memory. /// \param RemainingBytes The number of bytes to copy. @@ -1296,7 +1330,8 @@ public: void getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const; + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicCpySize = None) const; /// \returns True if the two functions have compatible attributes for inlining /// purposes. @@ -1536,8 +1571,8 @@ public: int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace, Instruction *I) = 0; - virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, - TargetTransformInfo::LSRCost &C2) = 0; + virtual bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2) = 0; virtual bool isNumRegsMajorCostOfLSR() = 0; virtual bool isProfitableLSRChainElement(Instruction *I) = 0; virtual bool canMacroFuseCmp() = 0; @@ -1550,6 +1585,8 @@ public: virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; + virtual bool isLegalBroadcastLoad(Type *ElementTy, + ElementCount NumElements) const = 0; virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) = 0; virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0; virtual bool forceScalarizeMaskedGather(VectorType *DataType, @@ -1558,6 +1595,9 @@ public: Align Alignment) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; + virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, + unsigned Opcode1, + const SmallBitVector &OpcodeMask) const = 0; virtual bool enableOrderedReductions() = 0; virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; @@ -1571,7 +1611,7 @@ public: virtual bool isProfitableToHoist(Instruction *I) = 0; virtual bool useAA() = 0; virtual bool isTypeLegal(Type *Ty) = 0; - virtual InstructionCost getRegUsageForType(Type *Ty) = 0; + virtual unsigned getRegUsageForType(Type *Ty) = 0; virtual bool shouldBuildLookupTables() = 0; virtual bool shouldBuildLookupTablesForConstant(Constant *C) = 0; virtual bool shouldBuildRelLookupTables() = 0; @@ -1584,6 +1624,7 @@ public: getOperandsScalarizationOverhead(ArrayRef<const Value *> Args, ArrayRef<Type *> Tys) = 0; virtual bool supportsEfficientVectorElementLoadStore() = 0; + virtual bool supportsTailCalls() = 0; virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0; virtual MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const = 0; @@ -1618,10 +1659,13 @@ public: virtual unsigned getMinVectorRegisterBitWidth() const = 0; virtual Optional<unsigned> getMaxVScale() const = 0; virtual Optional<unsigned> getVScaleForTuning() const = 0; - virtual bool shouldMaximizeVectorBandwidth() const = 0; + virtual bool + shouldMaximizeVectorBandwidth(TargetTransformInfo::RegisterKind K) const = 0; virtual ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const = 0; virtual unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const = 0; + virtual unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, + Type *ScalarValTy) const = 0; virtual bool shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) = 0; virtual unsigned getCacheLineSize() const = 0; @@ -1660,7 +1704,8 @@ public: ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr) = 0; virtual InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask, int Index, - VectorType *SubTp) = 0; + VectorType *SubTp, + ArrayRef<const Value *> Args) = 0; virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, TTI::TargetCostKind CostKind, @@ -1734,15 +1779,17 @@ public: virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0; virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) = 0; - virtual Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, - unsigned SrcAddrSpace, - unsigned DestAddrSpace, - unsigned SrcAlign, - unsigned DestAlign) const = 0; + virtual Type * + getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, unsigned DestAddrSpace, + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicElementSize) const = 0; + virtual void getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const = 0; + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicCpySize) const = 0; virtual bool areInlineCompatible(const Function *Caller, const Function *Callee) const = 0; virtual bool areTypesABICompatible(const Function *Caller, @@ -1920,8 +1967,8 @@ public: return Impl.isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace, I); } - bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, - TargetTransformInfo::LSRCost &C2) override { + bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2) override { return Impl.isLSRCostLess(C1, C2); } bool isNumRegsMajorCostOfLSR() override { @@ -1953,6 +2000,10 @@ public: bool isLegalNTLoad(Type *DataType, Align Alignment) override { return Impl.isLegalNTLoad(DataType, Alignment); } + bool isLegalBroadcastLoad(Type *ElementTy, + ElementCount NumElements) const override { + return Impl.isLegalBroadcastLoad(ElementTy, NumElements); + } bool isLegalMaskedScatter(Type *DataType, Align Alignment) override { return Impl.isLegalMaskedScatter(DataType, Alignment); } @@ -1973,6 +2024,10 @@ public: bool isLegalMaskedExpandLoad(Type *DataType) override { return Impl.isLegalMaskedExpandLoad(DataType); } + bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, + const SmallBitVector &OpcodeMask) const override { + return Impl.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask); + } bool enableOrderedReductions() override { return Impl.enableOrderedReductions(); } @@ -2001,7 +2056,7 @@ public: } bool useAA() override { return Impl.useAA(); } bool isTypeLegal(Type *Ty) override { return Impl.isTypeLegal(Ty); } - InstructionCost getRegUsageForType(Type *Ty) override { + unsigned getRegUsageForType(Type *Ty) override { return Impl.getRegUsageForType(Ty); } bool shouldBuildLookupTables() override { @@ -2032,6 +2087,8 @@ public: return Impl.supportsEfficientVectorElementLoadStore(); } + bool supportsTailCalls() override { return Impl.supportsTailCalls(); } + bool enableAggressiveInterleaving(bool LoopHasReductions) override { return Impl.enableAggressiveInterleaving(LoopHasReductions); } @@ -2108,8 +2165,9 @@ public: Optional<unsigned> getVScaleForTuning() const override { return Impl.getVScaleForTuning(); } - bool shouldMaximizeVectorBandwidth() const override { - return Impl.shouldMaximizeVectorBandwidth(); + bool shouldMaximizeVectorBandwidth( + TargetTransformInfo::RegisterKind K) const override { + return Impl.shouldMaximizeVectorBandwidth(K); } ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const override { @@ -2118,6 +2176,10 @@ public: unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override { return Impl.getMaximumVF(ElemWidth, Opcode); } + unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, + Type *ScalarValTy) const override { + return Impl.getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); + } bool shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) override { return Impl.shouldConsiderAddressTypePromotion( @@ -2180,8 +2242,9 @@ public: } InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask, int Index, - VectorType *SubTp) override { - return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp); + VectorType *SubTp, + ArrayRef<const Value *> Args) override { + return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp, Args); } InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, @@ -2298,20 +2361,22 @@ public: Type *ExpectedType) override { return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } - Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, - unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, - unsigned DestAlign) const override { + Type *getMemcpyLoopLoweringType( + LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicElementSize) const override { return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, - DestAddrSpace, SrcAlign, DestAlign); + DestAddrSpace, SrcAlign, DestAlign, + AtomicElementSize); } void getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const override { + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicCpySize) const override { Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, - SrcAlign, DestAlign); + SrcAlign, DestAlign, AtomicCpySize); } bool areInlineCompatible(const Function *Caller, const Function *Callee) const override { diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index a32744f8d58b..a70c418974f5 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -18,18 +18,16 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/Function.h" #include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" -#include "llvm/IR/Type.h" #include <utility> -using namespace llvm::PatternMatch; - namespace llvm { +class Function; + /// Base class for use as a mix-in that aids implementing /// a TargetTransformInfo-compatible class. class TargetTransformInfoImplBase { @@ -212,7 +210,7 @@ public: return !BaseGV && BaseOffset == 0 && (Scale == 0 || Scale == 1); } - bool isLSRCostLess(TTI::LSRCost &C1, TTI::LSRCost &C2) const { + bool isLSRCostLess(const TTI::LSRCost &C1, const TTI::LSRCost &C2) const { return std::tie(C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < std::tie(C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, C2.NumBaseAdds, @@ -258,6 +256,10 @@ public: return Alignment >= DataSize && isPowerOf2_32(DataSize); } + bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const { + return false; + } + bool isLegalMaskedScatter(Type *DataType, Align Alignment) const { return false; } @@ -277,6 +279,11 @@ public: bool isLegalMaskedCompressStore(Type *DataType) const { return false; } + bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, + const SmallBitVector &OpcodeMask) const { + return false; + } + bool isLegalMaskedExpandLoad(Type *DataType) const { return false; } bool enableOrderedReductions() const { return false; } @@ -310,7 +317,7 @@ public: bool isTypeLegal(Type *Ty) const { return false; } - InstructionCost getRegUsageForType(Type *Ty) const { return 1; } + unsigned getRegUsageForType(Type *Ty) const { return 1; } bool shouldBuildLookupTables() const { return true; } @@ -333,6 +340,8 @@ public: bool supportsEfficientVectorElementLoadStore() const { return false; } + bool supportsTailCalls() const { return true; } + bool enableAggressiveInterleaving(bool LoopHasReductions) const { return false; } @@ -415,13 +424,17 @@ public: Optional<unsigned> getMaxVScale() const { return None; } Optional<unsigned> getVScaleForTuning() const { return None; } - bool shouldMaximizeVectorBandwidth() const { return false; } + bool + shouldMaximizeVectorBandwidth(TargetTransformInfo::RegisterKind K) const { + return false; + } ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const { return ElementCount::get(0, IsScalable); } unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const { return 0; } + unsigned getStoreMinimumVF(unsigned VF, Type *, Type *) const { return VF; } bool shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { @@ -490,7 +503,8 @@ public: InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, int Index, - VectorType *SubTp) const { + VectorType *SubTp, + ArrayRef<const Value *> Args = None) const { return 1; } @@ -697,16 +711,21 @@ public: Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const { - return Type::getInt8Ty(Context); + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicElementSize) const { + return AtomicElementSize ? Type::getIntNTy(Context, *AtomicElementSize * 8) + : Type::getInt8Ty(Context); } void getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const { - for (unsigned i = 0; i != RemainingBytes; ++i) - OpsOut.push_back(Type::getInt8Ty(Context)); + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicCpySize) const { + unsigned OpSizeInBytes = AtomicCpySize ? *AtomicCpySize : 1; + Type *OpType = Type::getIntNTy(Context, OpSizeInBytes * 8); + for (unsigned i = 0; i != RemainingBytes; i += OpSizeInBytes) + OpsOut.push_back(OpType); } bool areInlineCompatible(const Function *Caller, @@ -960,6 +979,8 @@ public: InstructionCost getUserCost(const User *U, ArrayRef<const Value *> Operands, TTI::TargetCostKind CostKind) { + using namespace llvm::PatternMatch; + auto *TargetTTI = static_cast<T *>(this); // Handle non-intrinsic calls, invokes, and callbr. // FIXME: Unlikely to be true for anything but CodeSize. @@ -976,8 +997,6 @@ public: } Type *Ty = U->getType(); - Type *OpTy = - U->getNumOperands() == 1 ? U->getOperand(0)->getType() : nullptr; unsigned Opcode = Operator::getOpcode(U); auto *I = dyn_cast<Instruction>(U); switch (Opcode) { @@ -1049,9 +1068,11 @@ public: case Instruction::FPExt: case Instruction::SExt: case Instruction::ZExt: - case Instruction::AddrSpaceCast: + case Instruction::AddrSpaceCast: { + Type *OpTy = U->getOperand(0)->getType(); return TargetTTI->getCastInstrCost( Opcode, Ty, OpTy, TTI::getCastContextHint(I), CostKind, I); + } case Instruction::Store: { auto *SI = cast<StoreInst>(U); Type *ValTy = U->getOperand(0)->getType(); @@ -1137,13 +1158,14 @@ public: if (Shuffle->isExtractSubvectorMask(SubIndex)) return TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, VecSrcTy, Shuffle->getShuffleMask(), SubIndex, - VecTy); + VecTy, Operands); if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex)) return TargetTTI->getShuffleCost( TTI::SK_InsertSubvector, VecTy, Shuffle->getShuffleMask(), SubIndex, - FixedVectorType::get(VecTy->getScalarType(), NumSubElts)); + FixedVectorType::get(VecTy->getScalarType(), NumSubElts), + Operands); int ReplicationFactor, VF; if (Shuffle->isReplicationMask(ReplicationFactor, VF)) { @@ -1166,31 +1188,37 @@ public: if (Shuffle->isReverse()) return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); if (Shuffle->isSelect()) return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); if (Shuffle->isTranspose()) return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); if (Shuffle->isZeroEltSplat()) return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); if (Shuffle->isSingleSource()) return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex)) return TargetTTI->getShuffleCost( TTI::SK_InsertSubvector, VecTy, Shuffle->getShuffleMask(), SubIndex, - FixedVectorType::get(VecTy->getScalarType(), NumSubElts)); + FixedVectorType::get(VecTy->getScalarType(), NumSubElts), Operands); return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy, - Shuffle->getShuffleMask(), 0, nullptr); + Shuffle->getShuffleMask(), 0, nullptr, + Operands); } case Instruction::ExtractElement: { auto *EEI = dyn_cast<ExtractElementInst>(U); diff --git a/llvm/include/llvm/Analysis/TensorSpec.h b/llvm/include/llvm/Analysis/TensorSpec.h new file mode 100644 index 000000000000..382ab3f10445 --- /dev/null +++ b/llvm/include/llvm/Analysis/TensorSpec.h @@ -0,0 +1,132 @@ +//===- TensorSpec.h - type descriptor for a tensor --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +#ifndef LLVM_ANALYSIS_TENSORSPEC_H +#define LLVM_ANALYSIS_TENSORSPEC_H + +#include "llvm/Config/llvm-config.h" + +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Support/JSON.h" + +#include <memory> +#include <vector> + +namespace llvm { +/// TensorSpec encapsulates the specification of a tensor: its dimensions, or +/// "shape" (row-major), its type (see TensorSpec::getDataType specializations +/// for supported types), its name and port (see "TensorFlow: Large-Scale +/// Machine Learning on Heterogeneous Distributed Systems", section 4.2, para 2: +/// https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45166.pdf) +/// +/// Known tensor types. The left part is the C type, the right is a name we +/// can use to identify the type (to implement TensorSpec equality checks), and +/// to use, if needed, when mapping to an underlying evaluator's type system. +/// The main requirement is that the C type we use has the same size and +/// encoding (e.g. endian-ness) as the one used by the evaluator. +#define SUPPORTED_TENSOR_TYPES(M) \ + M(float, Float) \ + M(double, Double) \ + M(int8_t, Int8) \ + M(uint8_t, UInt8) \ + M(int16_t, Int16) \ + M(uint16_t, UInt16) \ + M(int32_t, Int32) \ + M(uint32_t, UInt32) \ + M(int64_t, Int64) \ + M(uint64_t, UInt64) + +enum class TensorType { + Invalid, +#define _TENSOR_TYPE_ENUM_MEMBERS(_, Name) Name, + SUPPORTED_TENSOR_TYPES(_TENSOR_TYPE_ENUM_MEMBERS) +#undef _TENSOR_TYPE_ENUM_MEMBERS +}; + +class TensorSpec final { +public: + template <typename T> + static TensorSpec createSpec(const std::string &Name, + const std::vector<int64_t> &Shape, + int Port = 0) { + return TensorSpec(Name, Port, getDataType<T>(), sizeof(T), Shape); + } + + const std::string &name() const { return Name; } + int port() const { return Port; } + TensorType type() const { return Type; } + const std::vector<int64_t> &shape() const { return Shape; } + + bool operator==(const TensorSpec &Other) const { + return Name == Other.Name && Port == Other.Port && Type == Other.Type && + Shape == Other.Shape; + } + + bool operator!=(const TensorSpec &Other) const { return !(*this == Other); } + + /// Get the number of elements in a tensor with this shape. + size_t getElementCount() const { return ElementCount; } + /// Get the size, in bytes, of one element. + size_t getElementByteSize() const { return ElementSize; } + /// Get the total size of a memory buffer needed to store the whole tensor. + size_t getTotalTensorBufferSize() const { return ElementCount * ElementSize; } + + template <typename T> bool isElementType() const { + return getDataType<T>() == Type; + } + +private: + TensorSpec(const std::string &Name, int Port, TensorType Type, + size_t ElementSize, const std::vector<int64_t> &Shape); + + template <typename T> static TensorType getDataType(); + + std::string Name; + int Port = 0; + TensorType Type = TensorType::Invalid; + std::vector<int64_t> Shape; + size_t ElementCount = 0; + size_t ElementSize = 0; +}; + +/// Construct a TensorSpec from a JSON dictionary of the form: +/// { "name": <string>, +/// "port": <int>, +/// "type": <string. Use LLVM's types, e.g. float, double, int64_t>, +/// "shape": <array of ints> } +/// For the "type" field, see the C++ primitive types used in +/// TFUTILS_SUPPORTED_TYPES. +Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx, + const json::Value &Value); + +struct LoggedFeatureSpec { + TensorSpec Spec; + Optional<std::string> LoggingName; + const std::string &getLoggingName() const { + return LoggingName ? *LoggingName : Spec.name(); + } +}; + +/// Load the output specs. If SpecFileOverride is not empty, that path is used. +/// Otherwise, the file is assumed to be called 'output_spec.json' and be found +/// under ModelPath (the model directory). +/// The first output tensor name must match ExpectedDecisionName. +/// In case of error, the return is None and the error is logged. +Optional<std::vector<LoggedFeatureSpec>> +loadOutputSpecs(LLVMContext &Ctx, StringRef ExpectedDecisionName, + StringRef ModelPath, StringRef SpecFileOverride = StringRef()); + +#define TFUTILS_GETDATATYPE_DEF(T, Name) \ + template <> TensorType TensorSpec::getDataType<T>(); +SUPPORTED_TENSOR_TYPES(TFUTILS_GETDATATYPE_DEF) + +#undef TFUTILS_GETDATATYPE_DEF +} // namespace llvm + +#endif // LLVM_ANALYSIS_TENSORSPEC_H diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h index 074c40942b06..dab67aad1ab0 100644 --- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h +++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h @@ -14,11 +14,11 @@ #ifndef LLVM_ANALYSIS_TYPEMETADATAUTILS_H #define LLVM_ANALYSIS_TYPEMETADATAUTILS_H -#include "llvm/ADT/SmallVector.h" #include <cstdint> namespace llvm { +template <typename T> class SmallVectorImpl; class CallBase; class CallInst; class Constant; diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h index 785b9fe949a5..372c35863f3f 100644 --- a/llvm/include/llvm/Analysis/Utils/TFUtils.h +++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h @@ -13,6 +13,7 @@ #ifdef LLVM_HAVE_TF_API #include "llvm/ADT/StringMap.h" +#include "llvm/Analysis/TensorSpec.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/JSON.h" @@ -38,86 +39,6 @@ namespace llvm { class TFModelEvaluatorImpl; class EvaluationResultImpl; -/// TensorSpec encapsulates the specification of a tensor: its dimensions, or -/// "shape" (row-major), its type (see TensorSpec::getDataType specializations -/// for supported types), its name and port (see "TensorFlow: Large-Scale -/// Machine Learning on Heterogeneous Distributed Systems", section 4.2, para 2: -/// https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45166.pdf) -/// -/// TensorSpec is used to set up a TFModelEvaluator by describing the expected -/// inputs and outputs. -class TensorSpec final { -public: - template <typename T> - static TensorSpec createSpec(const std::string &Name, - const std::vector<int64_t> &Shape, - int Port = 0) { - return TensorSpec(Name, Port, getDataType<T>(), Shape); - } - - const std::string &name() const { return Name; } - int port() const { return Port; } - int typeIndex() const { return TypeIndex; } - const std::vector<int64_t> &shape() const { return Shape; } - - bool operator==(const TensorSpec &Other) const { - return Name == Other.Name && Port == Other.Port && - TypeIndex == Other.TypeIndex && Shape == Other.Shape; - } - - bool operator!=(const TensorSpec &Other) const { return !(*this == Other); } - - /// Get the number of elements in a tensor with this shape. - size_t getElementCount() const { return ElementCount; } - /// Get the size, in bytes, of one element. - size_t getElementByteSize() const; - - template <typename T> bool isElementType() const { - return getDataType<T>() == TypeIndex; - } - -private: - TensorSpec(const std::string &Name, int Port, int TypeIndex, - const std::vector<int64_t> &Shape); - - template <typename T> static int getDataType() { - llvm_unreachable("Undefined tensor type"); - } - - std::string Name; - int Port = 0; - int TypeIndex = 0; - std::vector<int64_t> Shape; - size_t ElementCount = 0; -}; - -/// Construct a TensorSpec from a JSON dictionary of the form: -/// { "name": <string>, -/// "port": <int>, -/// "type": <string. Use LLVM's types, e.g. float, double, int64_t>, -/// "shape": <array of ints> } -/// For the "type" field, see the C++ primitive types used in -/// TFUTILS_SUPPORTED_TYPES. -Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx, - const json::Value &Value); - -struct LoggedFeatureSpec { - TensorSpec Spec; - Optional<std::string> LoggingName; - const std::string &getLoggingName() const { - return LoggingName ? *LoggingName : Spec.name(); - } -}; - -/// Load the output specs. If SpecFileOverride is not empty, that path is used. -/// Otherwise, the file is assumed to be called 'output_spec.json' and be found -/// under ModelPath (the model directory). -/// The first output tensor name must match ExpectedDecisionName. -/// In case of error, the return is None and the error is logged. -Optional<std::vector<LoggedFeatureSpec>> -loadOutputSpecs(LLVMContext &Ctx, StringRef ExpectedDecisionName, - StringRef ModelPath, StringRef SpecFileOverride = StringRef()); - /// Logging utility - given an ordered specification of features, and assuming /// a scalar reward, allow logging feature values and rewards, and then print /// as tf.train.SequenceExample text protobuf. @@ -262,27 +183,6 @@ private: std::unique_ptr<TFModelEvaluatorImpl> Impl; }; -/// List of supported types, as a pair: -/// - C++ type -/// - enum name (implementation-specific) -#define TFUTILS_SUPPORTED_TYPES(M) \ - M(float, TF_FLOAT) \ - M(double, TF_DOUBLE) \ - M(int8_t, TF_INT8) \ - M(uint8_t, TF_UINT8) \ - M(int16_t, TF_INT16) \ - M(uint16_t, TF_UINT16) \ - M(int32_t, TF_INT32) \ - M(uint32_t, TF_UINT32) \ - M(int64_t, TF_INT64) \ - M(uint64_t, TF_UINT64) - -#define TFUTILS_GETDATATYPE_DEF(T, E) \ - template <> int TensorSpec::getDataType<T>(); - -TFUTILS_SUPPORTED_TYPES(TFUTILS_GETDATATYPE_DEF) - -#undef TFUTILS_GETDATATYPE_DEF } // namespace llvm #endif // LLVM_HAVE_TF_API diff --git a/llvm/include/llvm/Analysis/ValueLattice.h b/llvm/include/llvm/Analysis/ValueLattice.h index 1b32fca50697..bc6b279e9ed5 100644 --- a/llvm/include/llvm/Analysis/ValueLattice.h +++ b/llvm/include/llvm/Analysis/ValueLattice.h @@ -9,16 +9,18 @@ #ifndef LLVM_ANALYSIS_VALUELATTICE_H #define LLVM_ANALYSIS_VALUELATTICE_H -#include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/ConstantRange.h" #include "llvm/IR/Instructions.h" -// + //===----------------------------------------------------------------------===// // ValueLatticeElement //===----------------------------------------------------------------------===// namespace llvm { +class Constant; + /// This class represents lattice values for constants. /// /// FIXME: This is basically just for bringup, this can be made a lot more rich diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 5b39b0244339..3b29bf1d53b4 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -21,12 +21,12 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Operator.h" #include <cassert> #include <cstdint> namespace llvm { +class Operator; class AddOperator; class AllocaInst; class APInt; @@ -463,15 +463,37 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6; const DominatorTree *DT = nullptr, const TargetLibraryInfo *TLI = nullptr); + /// This returns the same result as isSafeToSpeculativelyExecute if Opcode is + /// the actual opcode of Inst. If the provided and actual opcode differ, the + /// function (virtually) overrides the opcode of Inst with the provided + /// Opcode. There are come constraints in this case: + /// * If Opcode has a fixed number of operands (eg, as binary operators do), + /// then Inst has to have at least as many leading operands. The function + /// will ignore all trailing operands beyond that number. + /// * If Opcode allows for an arbitrary number of operands (eg, as CallInsts + /// do), then all operands are considered. + /// * The virtual instruction has to satisfy all typing rules of the provided + /// Opcode. + /// * This function is pessimistic in the following sense: If one actually + /// materialized the virtual instruction, then isSafeToSpeculativelyExecute + /// may say that the materialized instruction is speculatable whereas this + /// function may have said that the instruction wouldn't be speculatable. + /// This behavior is a shortcoming in the current implementation and not + /// intentional. + bool isSafeToSpeculativelyExecuteWithOpcode( + unsigned Opcode, const Operator *Inst, const Instruction *CtxI = nullptr, + const DominatorTree *DT = nullptr, + const TargetLibraryInfo *TLI = nullptr); + /// Returns true if the result or effects of the given instructions \p I - /// depend on or influence global memory. - /// Memory dependence arises for example if the instruction reads from - /// memory or may produce effects or undefined behaviour. Memory dependent - /// instructions generally cannot be reorderd with respect to other memory - /// dependent instructions or moved into non-dominated basic blocks. - /// Instructions which just compute a value based on the values of their - /// operands are not memory dependent. - bool mayBeMemoryDependent(const Instruction &I); + /// depend values not reachable through the def use graph. + /// * Memory dependence arises for example if the instruction reads from + /// memory or may produce effects or undefined behaviour. Memory dependent + /// instructions generally cannot be reorderd with respect to other memory + /// dependent instructions. + /// * Control dependence arises for example if the instruction may fault + /// if lifted above a throwing call or infinite loop. + bool mayHaveNonDefUseDependency(const Instruction &I); /// Return true if it is an intrinsic that cannot be speculated but also /// cannot trap. diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 751c88a4ecbb..0005874ba040 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -236,7 +236,7 @@ class VFDatabase { // ensuring that the variant described in the attribute has a // corresponding definition or declaration of the vector // function in the Module M. - if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) { + if (Shape && (Shape.getValue().ScalarName == ScalarName)) { assert(CI.getModule()->getFunction(Shape.getValue().VectorName) && "Vector function is missing."); Mappings.push_back(Shape.getValue()); @@ -309,16 +309,16 @@ inline Type *ToVectorTy(Type *Scalar, unsigned VF) { /// Identify if the intrinsic is trivially vectorizable. /// This method returns true if the intrinsic's argument types are all scalars /// for the scalar form of the intrinsic and all vectors (or scalars handled by -/// hasVectorInstrinsicScalarOpd) for the vector form of the intrinsic. +/// isVectorIntrinsicWithScalarOpAtArg) for the vector form of the intrinsic. bool isTriviallyVectorizable(Intrinsic::ID ID); /// Identifies if the vector form of the intrinsic has a scalar operand. -bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx); +bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, + unsigned ScalarOpdIdx); -/// Identifies if the vector form of the intrinsic has a scalar operand that has +/// Identifies if the vector form of the intrinsic has a operand that has /// an overloaded type. -bool hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, - unsigned ScalarOpdIdx); +bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, unsigned OpdIdx); /// Returns intrinsic ID for call. /// For the input call instruction it finds mapping intrinsic and returns @@ -398,6 +398,24 @@ void narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask, bool widenShuffleMaskElts(int Scale, ArrayRef<int> Mask, SmallVectorImpl<int> &ScaledMask); +/// Splits and processes shuffle mask depending on the number of input and +/// output registers. The function does 2 main things: 1) splits the +/// source/destination vectors into real registers; 2) do the mask analysis to +/// identify which real registers are permuted. Then the function processes +/// resulting registers mask using provided action items. If no input register +/// is defined, \p NoInputAction action is used. If only 1 input register is +/// used, \p SingleInputAction is used, otherwise \p ManyInputsAction is used to +/// process > 2 input registers and masks. +/// \param Mask Original shuffle mask. +/// \param NumOfSrcRegs Number of source registers. +/// \param NumOfDestRegs Number of destination registers. +/// \param NumOfUsedRegs Number of actually used destination registers. +void processShuffleMasks( + ArrayRef<int> Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, + unsigned NumOfUsedRegs, function_ref<void()> NoInputAction, + function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction, + function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction); + /// Compute a map of integer instructions to their minimum legal type /// size. /// diff --git a/llvm/include/llvm/AsmParser/LLLexer.h b/llvm/include/llvm/AsmParser/LLLexer.h index c30165e4a97b..7bcb33f18768 100644 --- a/llvm/include/llvm/AsmParser/LLLexer.h +++ b/llvm/include/llvm/AsmParser/LLLexer.h @@ -37,7 +37,7 @@ namespace llvm { lltok::Kind CurKind; std::string StrVal; unsigned UIntVal; - Type *TyVal; + Type *TyVal = nullptr; APFloat APFloatVal; APSInt APSIntVal; diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h index 62af3afbc142..3389475b2c9a 100644 --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -14,18 +14,25 @@ #define LLVM_ASMPARSER_LLPARSER_H #include "LLLexer.h" -#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringMap.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/Attributes.h" +#include "llvm/IR/FMF.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/ModuleSummaryIndex.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/Type.h" #include <map> namespace llvm { class Module; + class ConstantRange; + class FunctionType; + class GlobalObject; + class SMDiagnostic; + class SMLoc; + class SourceMgr; + class Type; + struct MaybeAlign; + template <typename T> class Optional; class Function; class Value; class BasicBlock; @@ -88,6 +95,8 @@ namespace llvm { typedef LLLexer::LocTy LocTy; private: LLVMContext &Context; + // Lexer to determine whether to use opaque pointers or not. + LLLexer OPLex; LLLexer Lex; // Module being parsed, null if we are only parsing summary index. Module *M; @@ -150,8 +159,9 @@ namespace llvm { LLParser(StringRef F, SourceMgr &SM, SMDiagnostic &Err, Module *M, ModuleSummaryIndex *Index, LLVMContext &Context, SlotMapping *Slots = nullptr) - : Context(Context), Lex(F, SM, Err, Context), M(M), Index(Index), - Slots(Slots), BlockAddressPFS(nullptr) {} + : Context(Context), OPLex(F, SM, Err, Context), + Lex(F, SM, Err, Context), M(M), Index(Index), Slots(Slots), + BlockAddressPFS(nullptr) {} bool Run( bool UpgradeDebugInfo, DataLayoutCallbackTy DataLayoutCallback = [](StringRef) { return None; }); @@ -263,6 +273,8 @@ namespace llvm { bool parseOptionalAlignment(MaybeAlign &Alignment, bool AllowParens = false); bool parseOptionalDerefAttrBytes(lltok::Kind AttrKind, uint64_t &Bytes); + bool parseOptionalUWTableKind(UWTableKind &Kind); + bool parseAllocKind(AllocFnKind &Kind); bool parseScopeAndOrdering(bool IsAtomic, SyncScope::ID &SSID, AtomicOrdering &Ordering); bool parseScope(SyncScope::ID &SSID); @@ -503,6 +515,7 @@ namespace llvm { bool parseGlobalValueVector(SmallVectorImpl<Constant *> &Elts, Optional<unsigned> *InRangeOp = nullptr); bool parseOptionalComdat(StringRef GlobalName, Comdat *&C); + bool parseSanitizer(GlobalVariable *GV); bool parseMetadataAsValue(Value *&V, PerFunctionState &PFS); bool parseValueAsMetadata(Metadata *&MD, const Twine &TypeMsg, PerFunctionState *PFS); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 78ebb35e0ea4..230a1662cc04 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -88,7 +88,6 @@ enum Kind { kw_triple, kw_source_filename, kw_unwind, - kw_deplibs, // FIXME: Remove in 4.0 kw_datalayout, kw_volatile, kw_atomic, @@ -112,7 +111,6 @@ enum Kind { kw_exact, kw_inbounds, kw_inrange, - kw_align, kw_addrspace, kw_section, kw_partition, @@ -121,7 +119,6 @@ enum Kind { kw_module, kw_asm, kw_sideeffect, - kw_alignstack, kw_inteldialect, kw_gc, kw_prefix, @@ -177,81 +174,12 @@ enum Kind { // Attributes: kw_attributes, - kw_allocsize, - kw_alwaysinline, - kw_argmemonly, - kw_sanitize_address, - kw_sanitize_hwaddress, - kw_sanitize_memtag, - kw_builtin, - kw_byval, - kw_inalloca, - kw_cold, - kw_convergent, - kw_dereferenceable, - kw_dereferenceable_or_null, - kw_disable_sanitizer_instrumentation, - kw_elementtype, - kw_inaccessiblememonly, - kw_inaccessiblemem_or_argmemonly, - kw_inlinehint, - kw_inreg, - kw_jumptable, - kw_minsize, - kw_naked, - kw_nest, - kw_noalias, - kw_noundef, - kw_nobuiltin, - kw_nocallback, - kw_nocapture, - kw_noduplicate, - kw_nofree, - kw_noimplicitfloat, - kw_noinline, - kw_norecurse, - kw_nonlazybind, - kw_nomerge, - kw_nonnull, - kw_noprofile, - kw_noredzone, - kw_noreturn, - kw_nosync, - kw_nocf_check, - kw_nounwind, - kw_nosanitize_coverage, - kw_null_pointer_is_valid, - kw_optforfuzzing, - kw_optnone, - kw_optsize, - kw_preallocated, - kw_readnone, - kw_readonly, - kw_returned, - kw_returns_twice, - kw_signext, - kw_speculatable, - kw_ssp, - kw_sspreq, - kw_sspstrong, - kw_safestack, - kw_shadowcallstack, - kw_sret, - kw_sanitize_thread, - kw_sanitize_memory, - kw_speculative_load_hardening, - kw_strictfp, - kw_swifterror, - kw_swiftself, - kw_swiftasync, - kw_uwtable, - kw_vscale_range, - kw_willreturn, - kw_writeonly, - kw_zeroext, - kw_immarg, - kw_byref, - kw_mustprogress, + kw_sync, + kw_async, +#define GET_ATTR_NAMES +#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \ + kw_##DISPLAY_NAME, +#include "llvm/IR/Attributes.inc" kw_type, kw_opaque, @@ -415,7 +343,6 @@ enum Kind { kw_param, kw_hotness, kw_unknown, - kw_hot, kw_critical, kw_relbf, kw_variable, @@ -464,6 +391,19 @@ enum Kind { kw_bit, kw_varFlags, + // GV's with __attribute__((no_sanitize("address"))), or things in + // -fsanitize-ignorelist when built with ASan. + kw_no_sanitize_address, + // GV's with __attribute__((no_sanitize("hwaddress"))), or things in + // -fsanitize-ignorelist when built with HWASan. + kw_no_sanitize_hwaddress, + // GV's with __attribute__((no_sanitize("memtag"))), or things in + // -fsanitize-ignorelist when built with memory tagging. + kw_no_sanitize_memtag, + // GV's where the clang++ frontend (when ASan is used) notes that this is + // dynamically initialized, and thus needs ODR detection. + kw_sanitize_address_dyninit, + // Unsigned Valued tokens (UIntVal). LabelID, // 42: GlobalID, // @42 diff --git a/llvm/include/llvm/AsmParser/Parser.h b/llvm/include/llvm/AsmParser/Parser.h index e1c7f746a335..6710ae6e358d 100644 --- a/llvm/include/llvm/AsmParser/Parser.h +++ b/llvm/include/llvm/AsmParser/Parser.h @@ -13,7 +13,9 @@ #ifndef LLVM_ASMPARSER_PARSER_H #define LLVM_ASMPARSER_PARSER_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLForwardCompat.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" #include <memory> diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h index e7dde986784f..fb563ff198ef 100644 --- a/llvm/include/llvm/BinaryFormat/COFF.h +++ b/llvm/include/llvm/BinaryFormat/COFF.h @@ -24,7 +24,6 @@ #include "llvm/Support/DataTypes.h" #include <cassert> -#include <cstring> namespace llvm { namespace COFF { @@ -731,6 +730,10 @@ inline bool isReservedSectionNumber(int32_t SectionNumber) { return SectionNumber <= 0; } +/// Encode section name based on string table offset. +/// The size of Out must be at least COFF::NameSize. +bool encodeSectionName(char *Out, uint64_t Offset); + } // End namespace COFF. } // End namespace llvm. diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h new file mode 100644 index 000000000000..9e912c7bd4ba --- /dev/null +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -0,0 +1,131 @@ +//===-- llvm/BinaryFormat/DXContainer.h - The DXBC file format --*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines manifest constants for the DXContainer object file format. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_DXCONTAINER_H +#define LLVM_BINARYFORMAT_DXCONTAINER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SwapByteOrder.h" + +#include <stdint.h> + +namespace llvm { + +// The DXContainer file format is arranged as a header and "parts". Semantically +// parts are similar to sections in other object file formats. The File format +// structure is roughly: + +// ┌────────────────────────────────┐ +// │ Header │ +// ├────────────────────────────────┤ +// │ Part │ +// ├────────────────────────────────┤ +// │ Part │ +// ├────────────────────────────────┤ +// │ ... │ +// └────────────────────────────────┘ + +namespace dxbc { + +struct Hash { + uint8_t Digest[16]; +}; + +enum class HashFlags : uint32_t { + None = 0, // No flags defined. + IncludesSource = 1, // This flag indicates that the shader hash was computed + // taking into account source information (-Zss) +}; + +struct ShaderHash { + uint32_t Flags; // DxilShaderHashFlags + uint8_t Digest[16]; + + void swapBytes() { sys::swapByteOrder(Flags); } +}; + +struct ContainerVersion { + uint16_t Major; + uint16_t Minor; + + void swapBytes() { + sys::swapByteOrder(Major); + sys::swapByteOrder(Minor); + } +}; + +struct Header { + uint8_t Magic[4]; // "DXBC" + Hash FileHash; + ContainerVersion Version; + uint32_t FileSize; + uint32_t PartCount; + + void swapBytes() { + Version.swapBytes(); + sys::swapByteOrder(FileSize); + sys::swapByteOrder(PartCount); + } + // Structure is followed by part offsets: uint32_t PartOffset[PartCount]; + // The offset is to a PartHeader, which is followed by the Part Data. +}; + +/// Use this type to describe the size and type of a DXIL container part. +struct PartHeader { + uint8_t Name[4]; + uint32_t Size; + + void swapBytes() { sys::swapByteOrder(Size); } + StringRef getName() const { + return StringRef(reinterpret_cast<const char *>(&Name[0]), 4); + } + // Structure is followed directly by part data: uint8_t PartData[PartSize]. +}; + +struct BitcodeHeader { + uint8_t Magic[4]; // ACSII "DXIL". + uint8_t MajorVersion; // DXIL version. + uint8_t MinorVersion; // DXIL version. + uint16_t Unused; + uint32_t Offset; // Offset to LLVM bitcode (from start of header). + uint32_t Size; // Size of LLVM bitcode (in bytes). + // Followed by uint8_t[BitcodeHeader.Size] at &BitcodeHeader + Header.Offset + + void swapBytes() { + sys::swapByteOrder(MinorVersion); + sys::swapByteOrder(MajorVersion); + sys::swapByteOrder(Offset); + sys::swapByteOrder(Size); + } +}; + +struct ProgramHeader { + uint8_t MinorVersion : 4; + uint8_t MajorVersion : 4; + uint8_t Unused; + uint16_t ShaderKind; + uint32_t Size; // Size in uint32_t words including this header. + BitcodeHeader Bitcode; + + void swapBytes() { + sys::swapByteOrder(ShaderKind); + sys::swapByteOrder(Size); + Bitcode.swapBytes(); + } +}; + +static_assert(sizeof(ProgramHeader) == 24, "ProgramHeader Size incorrect!"); + +} // namespace dxbc +} // namespace llvm + +#endif // LLVM_BINARYFORMAT_DXCONTAINER_H diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h index 4473f506d371..e288c5191bdb 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -320,6 +320,10 @@ inline bool isFortran(SourceLanguage S) { return result; } +inline TypeKind getArrayIndexTypeEncoding(SourceLanguage S) { + return isFortran(S) ? DW_ATE_signed : DW_ATE_unsigned; +} + enum CaseSensitivity { // Identifier case codes DW_ID_case_sensitive = 0x00, diff --git a/llvm/include/llvm/BinaryFormat/DynamicTags.def b/llvm/include/llvm/BinaryFormat/DynamicTags.def index 814d8b113ec4..ae25ec53813c 100644 --- a/llvm/include/llvm/BinaryFormat/DynamicTags.def +++ b/llvm/include/llvm/BinaryFormat/DynamicTags.def @@ -209,6 +209,7 @@ MIPS_DYNAMIC_TAG(MIPS_RWPLT, 0x70000034) // Points to the base // of a writable PLT. MIPS_DYNAMIC_TAG(MIPS_RLD_MAP_REL, 0x70000035) // Relative offset of run time loader // map, used for debugging. +MIPS_DYNAMIC_TAG(MIPS_XHASH, 0x70000036) // GNU-style hash table with xlat. // PPC specific dynamic table entries. PPC_DYNAMIC_TAG(PPC_GOT, 0x70000000) // Uses Secure PLT ABI. diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 5d3b1270b538..1e0ef613788d 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -319,6 +319,7 @@ enum { EM_BPF = 247, // Linux kernel bpf virtual machine EM_VE = 251, // NEC SX-Aurora VE EM_CSKY = 252, // C-SKY 32-bit processor + EM_LOONGARCH = 258, // LoongArch }; // Object file classes. @@ -563,6 +564,15 @@ enum : unsigned { EF_MIPS_ARCH = 0xf0000000 // Mask for applying EF_MIPS_ARCH_ variant }; +// MIPS-specific section indexes +enum { + SHN_MIPS_ACOMMON = 0xff00, // Common symbols which are defined and allocated + SHN_MIPS_TEXT = 0xff01, // Not ABI compliant + SHN_MIPS_DATA = 0xff02, // Not ABI compliant + SHN_MIPS_SCOMMON = 0xff03, // Common symbols for global data area + SHN_MIPS_SUNDEFINED = 0xff04 // Undefined symbols for global data area +}; + // ELF Relocation types for Mips enum { #include "ELFRelocs/Mips.def" @@ -753,16 +763,18 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX1035 = 0x03d, EF_AMDGPU_MACH_AMDGCN_GFX1034 = 0x03e, EF_AMDGPU_MACH_AMDGCN_GFX90A = 0x03f, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X40 = 0x040, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X41 = 0x041, + EF_AMDGPU_MACH_AMDGCN_GFX940 = 0x040, + EF_AMDGPU_MACH_AMDGCN_GFX1100 = 0x041, EF_AMDGPU_MACH_AMDGCN_GFX1013 = 0x042, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X43 = 0x043, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X44 = 0x044, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X45 = 0x045, + EF_AMDGPU_MACH_AMDGCN_GFX1103 = 0x044, + EF_AMDGPU_MACH_AMDGCN_GFX1036 = 0x045, + EF_AMDGPU_MACH_AMDGCN_GFX1101 = 0x046, + EF_AMDGPU_MACH_AMDGCN_GFX1102 = 0x047, // First/last AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, - EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_RESERVED_0X45, + EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX1102, // Indicates if the "xnack" target feature is enabled for all code contained // in the object. @@ -865,12 +877,34 @@ enum { #include "ELFRelocs/VE.def" }; +// CSKY Specific e_flags +enum : unsigned { + EF_CSKY_801 = 0xa, + EF_CSKY_802 = 0x10, + EF_CSKY_803 = 0x9, + EF_CSKY_805 = 0x11, + EF_CSKY_807 = 0x6, + EF_CSKY_810 = 0x8, + EF_CSKY_860 = 0xb, + EF_CSKY_800 = 0x1f, + EF_CSKY_FLOAT = 0x2000, + EF_CSKY_DSP = 0x4000, + EF_CSKY_ABIV2 = 0x20000000, + EF_CSKY_EFV1 = 0x1000000, + EF_CSKY_EFV2 = 0x2000000, + EF_CSKY_EFV3 = 0x3000000 +}; // ELF Relocation types for CSKY enum { #include "ELFRelocs/CSKY.def" }; +// ELF Relocation types for LoongArch +enum { +#include "ELFRelocs/LoongArch.def" +}; + #undef ELF_RELOC // Section header. @@ -947,12 +981,15 @@ enum : unsigned { SHT_LLVM_ADDRSIG = 0x6fff4c03, // List of address-significant symbols // for safe ICF. SHT_LLVM_DEPENDENT_LIBRARIES = - 0x6fff4c04, // LLVM Dependent Library Specifiers. - SHT_LLVM_SYMPART = 0x6fff4c05, // Symbol partition specification. - SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition. - SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition. - SHT_LLVM_BB_ADDR_MAP = 0x6fff4c08, // LLVM Basic Block Address Map. + 0x6fff4c04, // LLVM Dependent Library Specifiers. + SHT_LLVM_SYMPART = 0x6fff4c05, // Symbol partition specification. + SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition. + SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition. + SHT_LLVM_BB_ADDR_MAP_V0 = + 0x6fff4c08, // LLVM Basic Block Address Map (old version kept for + // backward-compatibility). SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c09, // LLVM Call Graph Profile. + SHT_LLVM_BB_ADDR_MAP = 0x6fff4c0a, // LLVM Basic Block Address Map. // Android's experimental support for SHT_RELR sections. // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512 SHT_ANDROID_RELR = 0x6fffff00, // Relocation entries; only offsets. @@ -985,6 +1022,8 @@ enum : unsigned { SHT_RISCV_ATTRIBUTES = 0x70000003U, + SHT_CSKY_ATTRIBUTES = 0x70000001U, + SHT_HIPROC = 0x7fffffff, // Highest processor arch-specific type. SHT_LOUSER = 0x80000000, // Lowest type reserved for applications. SHT_HIUSER = 0xffffffff // Highest type reserved for applications. @@ -1036,6 +1075,9 @@ enum : unsigned { SHF_MASKOS = 0x0ff00000, + // Solaris equivalent of SHF_GNU_RETAIN. + SHF_SUNW_NODISCARD = 0x00100000, + // Bits indicating processor-specific flags. SHF_MASKPROC = 0xf0000000, @@ -1329,6 +1371,9 @@ enum { PT_MIPS_RTPROC = 0x70000001, // Runtime procedure table. PT_MIPS_OPTIONS = 0x70000002, // Options segment. PT_MIPS_ABIFLAGS = 0x70000003, // Abiflags segment. + + // RISCV program header types. + PT_RISCV_ATTRIBUTES = 0x70000003, }; // Segment flag bits. @@ -1531,6 +1576,31 @@ enum { NT_GNU_PROPERTY_TYPE_0 = 5, }; +// Android note types. +enum { + NT_ANDROID_TYPE_IDENT = 1, + NT_ANDROID_TYPE_KUSER = 3, + NT_ANDROID_TYPE_MEMTAG = 4, +}; + +// Memory tagging values used in NT_ANDROID_TYPE_MEMTAG notes. +enum { + // Enumeration to determine the tagging mode. In Android-land, 'SYNC' means + // running all threads in MTE Synchronous mode, and 'ASYNC' means to use the + // kernels auto-upgrade feature to allow for either MTE Asynchronous, + // Asymmetric, or Synchronous mode. This allows silicon vendors to specify, on + // a per-cpu basis what 'ASYNC' should mean. Generally, the expectation is + // "pick the most precise mode that's very fast". + NT_MEMTAG_LEVEL_NONE = 0, + NT_MEMTAG_LEVEL_ASYNC = 1, + NT_MEMTAG_LEVEL_SYNC = 2, + NT_MEMTAG_LEVEL_MASK = 3, + // Bits indicating whether the loader should prepare for MTE to be enabled on + // the heap and/or stack. + NT_MEMTAG_HEAP = 4, + NT_MEMTAG_STACK = 8, +}; + // Property types used in GNU_PROPERTY_TYPE_0 notes. enum : unsigned { GNU_PROPERTY_STACK_SIZE = 1, diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def new file mode 100644 index 000000000000..8cbfe2fe4235 --- /dev/null +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def @@ -0,0 +1,62 @@ +#ifndef ELF_RELOC +#error "ELF_RELOC must be defined" +#endif + +// These types and values are from the LoongArch ELF psABI which can be found at +// https://github.com/loongson/LoongArch-Documentation +// and these definitions has been adopted by binutils (include/elf/loongarch.h). +// The commit hash (main branch) we reference is: +// 9b3bd9f4a497115913c22f1a2a47863798fbc02a + +ELF_RELOC(R_LARCH_NONE, 0) +ELF_RELOC(R_LARCH_32, 1) +ELF_RELOC(R_LARCH_64, 2) +ELF_RELOC(R_LARCH_RELATIVE, 3) +ELF_RELOC(R_LARCH_COPY, 4) +ELF_RELOC(R_LARCH_JUMP_SLOT, 5) +ELF_RELOC(R_LARCH_TLS_DTPMOD32, 6) +ELF_RELOC(R_LARCH_TLS_DTPMOD64, 7) +ELF_RELOC(R_LARCH_TLS_DTPREL32, 8) +ELF_RELOC(R_LARCH_TLS_DTPREL64, 9) +ELF_RELOC(R_LARCH_TLS_TPREL32, 10) +ELF_RELOC(R_LARCH_TLS_TPREL64, 11) +ELF_RELOC(R_LARCH_IRELATIVE, 12) +ELF_RELOC(R_LARCH_MARK_LA, 20) +ELF_RELOC(R_LARCH_MARK_PCREL, 21) +ELF_RELOC(R_LARCH_SOP_PUSH_PCREL, 22) +ELF_RELOC(R_LARCH_SOP_PUSH_ABSOLUTE, 23) +ELF_RELOC(R_LARCH_SOP_PUSH_DUP, 24) +ELF_RELOC(R_LARCH_SOP_PUSH_GPREL, 25) +ELF_RELOC(R_LARCH_SOP_PUSH_TLS_TPREL, 26) +ELF_RELOC(R_LARCH_SOP_PUSH_TLS_GOT, 27) +ELF_RELOC(R_LARCH_SOP_PUSH_TLS_GD, 28) +ELF_RELOC(R_LARCH_SOP_PUSH_PLT_PCREL, 29) +ELF_RELOC(R_LARCH_SOP_ASSERT, 30) +ELF_RELOC(R_LARCH_SOP_NOT, 31) +ELF_RELOC(R_LARCH_SOP_SUB, 32) +ELF_RELOC(R_LARCH_SOP_SL, 33) +ELF_RELOC(R_LARCH_SOP_SR, 34) +ELF_RELOC(R_LARCH_SOP_ADD, 35) +ELF_RELOC(R_LARCH_SOP_AND, 36) +ELF_RELOC(R_LARCH_SOP_IF_ELSE, 37) +ELF_RELOC(R_LARCH_SOP_POP_32_S_10_5, 38) +ELF_RELOC(R_LARCH_SOP_POP_32_U_10_12, 39) +ELF_RELOC(R_LARCH_SOP_POP_32_S_10_12, 40) +ELF_RELOC(R_LARCH_SOP_POP_32_S_10_16, 41) +ELF_RELOC(R_LARCH_SOP_POP_32_S_10_16_S2, 42) +ELF_RELOC(R_LARCH_SOP_POP_32_S_5_20, 43) +ELF_RELOC(R_LARCH_SOP_POP_32_S_0_5_10_16_S2, 44) +ELF_RELOC(R_LARCH_SOP_POP_32_S_0_10_10_16_S2, 45) +ELF_RELOC(R_LARCH_SOP_POP_32_U, 46) +ELF_RELOC(R_LARCH_ADD8, 47) +ELF_RELOC(R_LARCH_ADD16, 48) +ELF_RELOC(R_LARCH_ADD24, 49) +ELF_RELOC(R_LARCH_ADD32, 50) +ELF_RELOC(R_LARCH_ADD64, 51) +ELF_RELOC(R_LARCH_SUB8, 52) +ELF_RELOC(R_LARCH_SUB16, 53) +ELF_RELOC(R_LARCH_SUB24, 54) +ELF_RELOC(R_LARCH_SUB32, 55) +ELF_RELOC(R_LARCH_SUB64, 56) +ELF_RELOC(R_LARCH_GNU_VTINHERIT, 57) +ELF_RELOC(R_LARCH_GNU_VTENTRY, 58) diff --git a/llvm/include/llvm/BinaryFormat/GOFF.h b/llvm/include/llvm/BinaryFormat/GOFF.h new file mode 100644 index 000000000000..96992414c6cc --- /dev/null +++ b/llvm/include/llvm/BinaryFormat/GOFF.h @@ -0,0 +1,33 @@ +//===-- llvm/BinaryFormat/GOFF.h - GOFF definitions --------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header contains common, non-processor-specific data structures and +// constants for the GOFF file format. +// +// GOFF specifics can be found in MVS Program Management: Advanced Facilities +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_GOFF_H +#define LLVM_BINARYFORMAT_GOFF_H + +#include "llvm/Support/DataTypes.h" + +namespace llvm { + +namespace GOFF { + +// \brief Subsections of the primary C_CODE section in the object file. +enum SubsectionKind : uint8_t { + SK_PPA1 = 2, +}; + +} // end namespace GOFF + +} // end namespace llvm + +#endif // LLVM_BINARYFORMAT_GOFF_H diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h index ce3a5c46e0d1..c05e79333d38 100644 --- a/llvm/include/llvm/BinaryFormat/MachO.h +++ b/llvm/include/llvm/BinaryFormat/MachO.h @@ -255,7 +255,8 @@ enum BindType { enum BindSpecialDylib { BIND_SPECIAL_DYLIB_SELF = 0, BIND_SPECIAL_DYLIB_MAIN_EXECUTABLE = -1, - BIND_SPECIAL_DYLIB_FLAT_LOOKUP = -2 + BIND_SPECIAL_DYLIB_FLAT_LOOKUP = -2, + BIND_SPECIAL_DYLIB_WEAK_LOOKUP = -3 }; enum { @@ -1001,6 +1002,27 @@ struct nlist_64 { uint64_t n_value; }; +/// Structs for dyld chained fixups. +/// dyld_chained_fixups_header is the data pointed to by LC_DYLD_CHAINED_FIXUPS +/// load command. +struct dyld_chained_fixups_header { + uint32_t fixups_version; ///< 0 + uint32_t starts_offset; ///< Offset of dyld_chained_starts_in_image. + uint32_t imports_offset; ///< Offset of imports table in chain_data. + uint32_t symbols_offset; ///< Offset of symbol strings in chain_data. + uint32_t imports_count; ///< Number of imported symbol names. + uint32_t imports_format; ///< DYLD_CHAINED_IMPORT* + uint32_t symbols_format; ///< 0 => uncompressed, 1 => zlib compressed +}; + +/// dyld_chained_starts_in_image is embedded in LC_DYLD_CHAINED_FIXUPS payload. +/// Each each seg_info_offset entry is the offset into this struct for that +/// segment followed by pool of dyld_chain_starts_in_segment data. +struct dyld_chained_starts_in_image { + uint32_t seg_count; + uint32_t seg_info_offset[1]; +}; + // Byte order swapping functions for MachO structs inline void swapStruct(fat_header &mh) { @@ -2008,6 +2030,16 @@ union alignas(4) macho_load_command { }; LLVM_PACKED_END +inline void swapStruct(dyld_chained_fixups_header &C) { + sys::swapByteOrder(C.fixups_version); + sys::swapByteOrder(C.starts_offset); + sys::swapByteOrder(C.imports_offset); + sys::swapByteOrder(C.symbols_offset); + sys::swapByteOrder(C.imports_count); + sys::swapByteOrder(C.imports_format); + sys::swapByteOrder(C.symbols_format); +} + /* code signing attributes of a process */ enum CodeSignAttrs { @@ -2205,6 +2237,17 @@ enum SecCSDigestAlgorithm { kSecCodeSignatureHashSHA512 = 5, /* SHA-512 */ }; +enum LinkerOptimizationHintKind { + LOH_ARM64_ADRP_ADRP = 1, + LOH_ARM64_ADRP_LDR = 2, + LOH_ARM64_ADRP_ADD_LDR = 3, + LOH_ARM64_ADRP_LDR_GOT_LDR = 4, + LOH_ARM64_ADRP_ADD_STR = 5, + LOH_ARM64_ADRP_LDR_GOT_STR = 6, + LOH_ARM64_ADRP_ADD = 7, + LOH_ARM64_ADRP_LDR_GOT = 8, +}; + } // end namespace MachO } // end namespace llvm diff --git a/llvm/include/llvm/BinaryFormat/Magic.h b/llvm/include/llvm/BinaryFormat/Magic.h index 6988b2dde656..c8e0dad42b0b 100644 --- a/llvm/include/llvm/BinaryFormat/Magic.h +++ b/llvm/include/llvm/BinaryFormat/Magic.h @@ -51,6 +51,9 @@ struct file_magic { wasm_object, ///< WebAssembly Object file pdb, ///< Windows PDB debug info file tapi_file, ///< Text-based Dynamic Library Stub file + cuda_fatbinary, ///< CUDA Fatbinary object file + offload_binary, ///< LLVM offload object file + dxcontainer_object, ///< DirectX container file }; bool is_object() const { return V != unknown; } diff --git a/llvm/include/llvm/BinaryFormat/Swift.def b/llvm/include/llvm/BinaryFormat/Swift.def index 6160e2551432..05b60e40632c 100644 --- a/llvm/include/llvm/BinaryFormat/Swift.def +++ b/llvm/include/llvm/BinaryFormat/Swift.def @@ -24,3 +24,10 @@ HANDLE_SWIFT_SECTION(builtin, "__swift5_builtin", "swift5_builtin", ".sw5bltn") HANDLE_SWIFT_SECTION(capture, "__swift5_capture", "swift5_capture", ".sw5cptr") HANDLE_SWIFT_SECTION(typeref, "__swift5_typeref", "swift5_typeref", ".sw5tyrf") HANDLE_SWIFT_SECTION(reflstr, "__swift5_reflstr", "swift5_reflstr", ".sw5rfst") +HANDLE_SWIFT_SECTION(conform, "__swift5_proto", "swift5_protocol_conformances", + ".sw5prtc$B") +HANDLE_SWIFT_SECTION(protocs, "__swift5_protos", "swift5_protocols", + ".sw5prt$B") +HANDLE_SWIFT_SECTION(acfuncs, "__swift5_acfuncs", "swift5_accessible_functions", + ".sw5acfn$B") +HANDLE_SWIFT_SECTION(mpenum, "__swift5_mpenum", "swift5_mpenum", ".sw5mpen$B") diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h index 0bc8c4e167d8..62a6881ef36a 100644 --- a/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/llvm/include/llvm/BinaryFormat/Wasm.h @@ -91,7 +91,7 @@ struct WasmTable { StringRef SymbolName; // from the "linking" section }; -struct WasmInitExpr { +struct WasmInitExprMVP { uint8_t Opcode; union { int32_t Int32; @@ -102,6 +102,13 @@ struct WasmInitExpr { } Value; }; +struct WasmInitExpr { + uint8_t Extended; // Set to non-zero if extended const is used (i.e. more than + // one instruction) + WasmInitExprMVP Inst; + ArrayRef<uint8_t> Body; +}; + struct WasmGlobalType { uint8_t Type; bool Mutable; @@ -245,7 +252,8 @@ enum : unsigned { WASM_SEC_CODE = 10, // Function bodies (code) WASM_SEC_DATA = 11, // Data segments WASM_SEC_DATACOUNT = 12, // Data segment count - WASM_SEC_TAG = 13 // Tag declarations + WASM_SEC_TAG = 13, // Tag declarations + WASM_SEC_LAST_KNOWN = WASM_SEC_TAG, }; // Type immediate encodings used in various contexts. @@ -276,6 +284,7 @@ enum : unsigned { WASM_OPCODE_CALL = 0x10, WASM_OPCODE_LOCAL_GET = 0x20, WASM_OPCODE_LOCAL_SET = 0x21, + WASM_OPCODE_LOCAL_TEE = 0x22, WASM_OPCODE_GLOBAL_GET = 0x23, WASM_OPCODE_GLOBAL_SET = 0x24, WASM_OPCODE_I32_STORE = 0x36, @@ -285,7 +294,11 @@ enum : unsigned { WASM_OPCODE_F32_CONST = 0x43, WASM_OPCODE_F64_CONST = 0x44, WASM_OPCODE_I32_ADD = 0x6a, + WASM_OPCODE_I32_SUB = 0x6b, + WASM_OPCODE_I32_MUL = 0x6c, WASM_OPCODE_I64_ADD = 0x7c, + WASM_OPCODE_I64_SUB = 0x7d, + WASM_OPCODE_I64_MUL = 0x7e, WASM_OPCODE_REF_NULL = 0xd0, }; @@ -458,8 +471,9 @@ inline bool operator==(const WasmTableType &LHS, const WasmTableType &RHS) { return LHS.ElemType == RHS.ElemType && LHS.Limits == RHS.Limits; } -std::string toString(WasmSymbolType type); -std::string relocTypetoString(uint32_t type); +llvm::StringRef toString(WasmSymbolType type); +llvm::StringRef relocTypetoString(uint32_t type); +llvm::StringRef sectionTypeToString(uint32_t type); bool relocTypeHasAddend(uint32_t type); } // end namespace wasm diff --git a/llvm/include/llvm/BinaryFormat/XCOFF.h b/llvm/include/llvm/BinaryFormat/XCOFF.h index cffd8618f1e3..5d23ec5cd911 100644 --- a/llvm/include/llvm/BinaryFormat/XCOFF.h +++ b/llvm/include/llvm/BinaryFormat/XCOFF.h @@ -54,6 +54,34 @@ enum AuxHeaderFlags64 : uint16_t { ///< future use and should be set to 0. }; +enum XCOFFInterpret : uint16_t { + OLD_XCOFF_INTERPRET = 1, + NEW_XCOFF_INTERPRET = 2 +}; + +enum FileFlag : uint16_t { + F_RELFLG = 0x0001, ///< relocation info stripped from file + F_EXEC = 0x0002, ///< file is executable (i.e., it + ///< has a loader section) + F_LNNO = 0x0004, ///< line numbers stripped from file + F_LSYMS = 0x0008, ///< local symbols stripped from file + F_FDPR_PROF = 0x0010, ///< file was profiled with FDPR + F_FDPR_OPTI = 0x0020, ///< file was reordered with FDPR + F_DSA = 0x0040, ///< file uses Dynamic Segment Allocation (32-bit + ///< only) + F_DEP_1 = 0x0080, ///< Data Execution Protection bit 1 + F_VARPG = 0x0100, ///< executable requests using variable size pages + F_LPTEXT = 0x0400, ///< executable requires large pages for text + F_LPDATA = 0x0800, ///< executable requires large pages for data + F_DYNLOAD = 0x1000, ///< file is dynamically loadable and + ///< executable (equivalent to F_EXEC on AIX) + F_SHROBJ = 0x2000, ///< file is a shared object + F_LOADONLY = + 0x4000, ///< file can be loaded by the system loader, but it is + ///< ignored by the linker if it is a member of an archive. + F_DEP_2 = 0x8000 ///< Data Execution Protection bit 2 +}; + // x_smclas field of x_csect from system header: /usr/include/syms.h /// Storage Mapping Class definitions. enum StorageMappingClass : uint8_t { @@ -212,6 +240,8 @@ enum VisibilityType : uint16_t { SYM_V_EXPORTED = 0x4000 }; +constexpr uint16_t VISIBILITY_MASK = 0x7000; + // Relocation types, defined in `/usr/include/reloc.h`. enum RelocationType : uint8_t { R_POS = 0x00, ///< Positive relocation. Provides the address of the referenced diff --git a/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h b/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h index f6fc284da33f..102e2257abcc 100644 --- a/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h +++ b/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h @@ -18,12 +18,13 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Support/Error.h" -#include "llvm/Support/raw_ostream.h" #include <map> #include <vector> namespace llvm { +class raw_ostream; + /// CurStreamTypeType - A type for CurStreamType enum CurStreamTypeType { UnknownBitstream, diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h index a82791c8720b..39ea48c33fc3 100644 --- a/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -15,12 +15,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Bitstream/BitCodes.h" -#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Bitstream/BitCodeEnums.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorOr.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include <cstdint> #include <memory> #include <string> @@ -30,6 +29,8 @@ namespace llvm { class LLVMContext; class Module; +class MemoryBuffer; +class ModuleSummaryIndex; typedef llvm::function_ref<Optional<std::string>(StringRef)> DataLayoutCallbackTy; diff --git a/llvm/include/llvm/Bitcode/BitcodeWriter.h b/llvm/include/llvm/Bitcode/BitcodeWriter.h index 96f25fce8ddb..248d33f4502e 100644 --- a/llvm/include/llvm/Bitcode/BitcodeWriter.h +++ b/llvm/include/llvm/Bitcode/BitcodeWriter.h @@ -17,7 +17,7 @@ #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include <map> #include <memory> #include <string> diff --git a/llvm/include/llvm/Bitcode/BitcodeWriterPass.h b/llvm/include/llvm/Bitcode/BitcodeWriterPass.h index dda5b20973c1..3c2471237532 100644 --- a/llvm/include/llvm/Bitcode/BitcodeWriterPass.h +++ b/llvm/include/llvm/Bitcode/BitcodeWriterPass.h @@ -14,7 +14,6 @@ #ifndef LLVM_BITCODE_BITCODEWRITERPASS_H #define LLVM_BITCODE_BITCODEWRITERPASS_H -#include "llvm/ADT/StringRef.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 6d0f51ce9c6d..5d96204ba42a 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -17,7 +17,10 @@ #ifndef LLVM_BITCODE_LLVMBITCODES_H #define LLVM_BITCODE_LLVMBITCODES_H -#include "llvm/Bitstream/BitCodes.h" +// This is the only file included, and it, in turn, is a leaf header. +// This allows external tools to dump the AST of this file and analyze it for +// changes without needing to fully or partially build LLVM itself. +#include "llvm/Bitstream/BitCodeEnums.h" namespace llvm { namespace bitc { @@ -582,14 +585,15 @@ enum FunctionCodes { 52, // CATCHSWITCH: [num,args...] or [num,args...,bb] // 53 is unused. // 54 is unused. - FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] - FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] - FUNC_CODE_INST_CALLBR = 57, // CALLBR: [attr, cc, norm, transfs, - // fnty, fnid, args...] - FUNC_CODE_INST_FREEZE = 58, // FREEZE: [opty, opval] - FUNC_CODE_INST_ATOMICRMW = 59, // ATOMICRMW: [ptrty, ptr, valty, val, - // operation, align, vol, - // ordering, synchscope] + FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] + FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] + FUNC_CODE_INST_CALLBR = 57, // CALLBR: [attr, cc, norm, transfs, + // fnty, fnid, args...] + FUNC_CODE_INST_FREEZE = 58, // FREEZE: [opty, opval] + FUNC_CODE_INST_ATOMICRMW = 59, // ATOMICRMW: [ptrty, ptr, valty, val, + // operation, align, vol, + // ordering, synchscope] + FUNC_CODE_BLOCKADDR_USERS = 60, // BLOCKADDR_USERS: [value...] }; enum UseListCodes { @@ -677,6 +681,11 @@ enum AttributeKindCodes { ATTR_KIND_NO_SANITIZE_COVERAGE = 76, ATTR_KIND_ELEMENTTYPE = 77, ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION = 78, + ATTR_KIND_NO_SANITIZE_BOUNDS = 79, + ATTR_KIND_ALLOC_ALIGN = 80, + ATTR_KIND_ALLOCATED_POINTER = 81, + ATTR_KIND_ALLOC_KIND = 82, + ATTR_KIND_PRESPLIT_COROUTINE = 83, }; enum ComdatSelectionKindCodes { diff --git a/llvm/include/llvm/Bitstream/BitCodeEnums.h b/llvm/include/llvm/Bitstream/BitCodeEnums.h new file mode 100644 index 000000000000..4288bd3987ae --- /dev/null +++ b/llvm/include/llvm/Bitstream/BitCodeEnums.h @@ -0,0 +1,90 @@ +//===- BitCodeEnums.h - Core enums for the bitstream format -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header defines "core" bitstream enum values. +// It has been separated from the other header that defines bitstream enum +// values, BitCodes.h, to allow tools to track changes to the various +// bitstream and bitcode enums without needing to fully or partially build +// LLVM itself. +// +// The enum values defined in this file should be considered permanent. If +// new features are added, they should have values added at the end of the +// respective lists. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BITSTREAM_BITCODEENUMS_H +#define LLVM_BITSTREAM_BITCODEENUMS_H + +namespace llvm { +/// Offsets of the 32-bit fields of bitstream wrapper header. +enum BitstreamWrapperHeader : unsigned { + BWH_MagicField = 0 * 4, + BWH_VersionField = 1 * 4, + BWH_OffsetField = 2 * 4, + BWH_SizeField = 3 * 4, + BWH_CPUTypeField = 4 * 4, + BWH_HeaderSize = 5 * 4 +}; + +namespace bitc { +enum StandardWidths { + BlockIDWidth = 8, // We use VBR-8 for block IDs. + CodeLenWidth = 4, // Codelen are VBR-4. + BlockSizeWidth = 32 // BlockSize up to 2^32 32-bit words = 16GB per block. +}; + +// The standard abbrev namespace always has a way to exit a block, enter a +// nested block, define abbrevs, and define an unabbreviated record. +enum FixedAbbrevIDs { + END_BLOCK = 0, // Must be zero to guarantee termination for broken bitcode. + ENTER_SUBBLOCK = 1, + + /// DEFINE_ABBREV - Defines an abbrev for the current block. It consists + /// of a vbr5 for # operand infos. Each operand info is emitted with a + /// single bit to indicate if it is a literal encoding. If so, the value is + /// emitted with a vbr8. If not, the encoding is emitted as 3 bits followed + /// by the info value as a vbr5 if needed. + DEFINE_ABBREV = 2, + + // UNABBREV_RECORDs are emitted with a vbr6 for the record code, followed by + // a vbr6 for the # operands, followed by vbr6's for each operand. + UNABBREV_RECORD = 3, + + // This is not a code, this is a marker for the first abbrev assignment. + FIRST_APPLICATION_ABBREV = 4 +}; + +/// StandardBlockIDs - All bitcode files can optionally include a BLOCKINFO +/// block, which contains metadata about other blocks in the file. +enum StandardBlockIDs { + /// BLOCKINFO_BLOCK is used to define metadata about blocks, for example, + /// standard abbrevs that should be available to all blocks of a specified + /// ID. + BLOCKINFO_BLOCK_ID = 0, + + // Block IDs 1-7 are reserved for future expansion. + FIRST_APPLICATION_BLOCKID = 8 +}; + +/// BlockInfoCodes - The blockinfo block contains metadata about user-defined +/// blocks. +enum BlockInfoCodes { + // DEFINE_ABBREV has magic semantics here, applying to the current SETBID'd + // block, instead of the BlockInfo block. + + BLOCKINFO_CODE_SETBID = 1, // SETBID: [blockid#] + BLOCKINFO_CODE_BLOCKNAME = 2, // BLOCKNAME: [name] + BLOCKINFO_CODE_SETRECORDNAME = 3 // BLOCKINFO_CODE_SETRECORDNAME: + // [id, name] +}; + +} // namespace bitc +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Bitstream/BitCodes.h b/llvm/include/llvm/Bitstream/BitCodes.h index 9cd4e535a470..93888f7d3b33 100644 --- a/llvm/include/llvm/Bitstream/BitCodes.h +++ b/llvm/include/llvm/Bitstream/BitCodes.h @@ -19,75 +19,12 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Bitstream/BitCodeEnums.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" #include <cassert> namespace llvm { -/// Offsets of the 32-bit fields of bitstream wrapper header. -enum BitstreamWrapperHeader : unsigned { - BWH_MagicField = 0 * 4, - BWH_VersionField = 1 * 4, - BWH_OffsetField = 2 * 4, - BWH_SizeField = 3 * 4, - BWH_CPUTypeField = 4 * 4, - BWH_HeaderSize = 5 * 4 -}; - -namespace bitc { - enum StandardWidths { - BlockIDWidth = 8, // We use VBR-8 for block IDs. - CodeLenWidth = 4, // Codelen are VBR-4. - BlockSizeWidth = 32 // BlockSize up to 2^32 32-bit words = 16GB per block. - }; - - // The standard abbrev namespace always has a way to exit a block, enter a - // nested block, define abbrevs, and define an unabbreviated record. - enum FixedAbbrevIDs { - END_BLOCK = 0, // Must be zero to guarantee termination for broken bitcode. - ENTER_SUBBLOCK = 1, - - /// DEFINE_ABBREV - Defines an abbrev for the current block. It consists - /// of a vbr5 for # operand infos. Each operand info is emitted with a - /// single bit to indicate if it is a literal encoding. If so, the value is - /// emitted with a vbr8. If not, the encoding is emitted as 3 bits followed - /// by the info value as a vbr5 if needed. - DEFINE_ABBREV = 2, - - // UNABBREV_RECORDs are emitted with a vbr6 for the record code, followed by - // a vbr6 for the # operands, followed by vbr6's for each operand. - UNABBREV_RECORD = 3, - - // This is not a code, this is a marker for the first abbrev assignment. - FIRST_APPLICATION_ABBREV = 4 - }; - - /// StandardBlockIDs - All bitcode files can optionally include a BLOCKINFO - /// block, which contains metadata about other blocks in the file. - enum StandardBlockIDs { - /// BLOCKINFO_BLOCK is used to define metadata about blocks, for example, - /// standard abbrevs that should be available to all blocks of a specified - /// ID. - BLOCKINFO_BLOCK_ID = 0, - - // Block IDs 1-7 are reserved for future expansion. - FIRST_APPLICATION_BLOCKID = 8 - }; - - /// BlockInfoCodes - The blockinfo block contains metadata about user-defined - /// blocks. - enum BlockInfoCodes { - // DEFINE_ABBREV has magic semantics here, applying to the current SETBID'd - // block, instead of the BlockInfo block. - - BLOCKINFO_CODE_SETBID = 1, // SETBID: [blockid#] - BLOCKINFO_CODE_BLOCKNAME = 2, // BLOCKNAME: [name] - BLOCKINFO_CODE_SETRECORDNAME = 3 // BLOCKINFO_CODE_SETRECORDNAME: - // [id, name] - }; - -} // End bitc namespace - /// BitCodeAbbrevOp - This describes one or more operands in an abbreviation. /// This is actually a union of two different things: /// 1. It could be a literal integer value ("the operand is always 17"). @@ -106,6 +43,10 @@ public: Blob = 5 // 32-bit aligned array of 8-bit characters. }; + static bool isValidEncoding(uint64_t E) { + return E >= 1 && E <= 5; + } + explicit BitCodeAbbrevOp(uint64_t V) : Val(V), IsLiteral(true) {} explicit BitCodeAbbrevOp(Encoding E, uint64_t Data = 0) : Val(Data), IsLiteral(false), Enc(E) {} @@ -179,6 +120,6 @@ public: OperandList.push_back(OpInfo); } }; -} // End llvm namespace +} // namespace llvm #endif diff --git a/llvm/include/llvm/Bitstream/BitstreamReader.h b/llvm/include/llvm/Bitstream/BitstreamReader.h index 37b7c4d73cff..10a0a4e0039e 100644 --- a/llvm/include/llvm/Bitstream/BitstreamReader.h +++ b/llvm/include/llvm/Bitstream/BitstreamReader.h @@ -19,7 +19,6 @@ #include "llvm/Bitstream/BitCodes.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBufferRef.h" #include <algorithm> #include <cassert> @@ -97,8 +96,6 @@ private: unsigned BitsInCurWord = 0; public: - static const constexpr size_t MaxChunkSize = sizeof(word_t) * 8; - SimpleBitstreamCursor() = default; explicit SimpleBitstreamCursor(ArrayRef<uint8_t> BitcodeBytes) : BitcodeBytes(BitcodeBytes) {} @@ -187,7 +184,7 @@ public: } Expected<word_t> Read(unsigned NumBits) { - static const unsigned BitsInWord = MaxChunkSize; + static const unsigned BitsInWord = sizeof(word_t) * 8; assert(NumBits && NumBits <= BitsInWord && "Cannot return zero or more than BitsInWord bits!"); @@ -229,24 +226,32 @@ public: return R; } - Expected<uint32_t> ReadVBR(unsigned NumBits) { + Expected<uint32_t> ReadVBR(const unsigned NumBits) { Expected<unsigned> MaybeRead = Read(NumBits); if (!MaybeRead) return MaybeRead; uint32_t Piece = MaybeRead.get(); - if ((Piece & (1U << (NumBits-1))) == 0) + assert(NumBits <= 32 && NumBits >= 1 && "Invalid NumBits value"); + const uint32_t MaskBitOrder = (NumBits - 1); + const uint32_t Mask = 1UL << MaskBitOrder; + + if ((Piece & Mask) == 0) return Piece; uint32_t Result = 0; unsigned NextBit = 0; while (true) { - Result |= (Piece & ((1U << (NumBits-1))-1)) << NextBit; + Result |= (Piece & (Mask - 1)) << NextBit; - if ((Piece & (1U << (NumBits-1))) == 0) + if ((Piece & Mask) == 0) return Result; NextBit += NumBits-1; + if (NextBit >= 32) + return createStringError(std::errc::illegal_byte_sequence, + "Unterminated VBR"); + MaybeRead = Read(NumBits); if (!MaybeRead) return MaybeRead; @@ -256,24 +261,31 @@ public: // Read a VBR that may have a value up to 64-bits in size. The chunk size of // the VBR must still be <= 32 bits though. - Expected<uint64_t> ReadVBR64(unsigned NumBits) { + Expected<uint64_t> ReadVBR64(const unsigned NumBits) { Expected<uint64_t> MaybeRead = Read(NumBits); if (!MaybeRead) return MaybeRead; uint32_t Piece = MaybeRead.get(); + assert(NumBits <= 32 && NumBits >= 1 && "Invalid NumBits value"); + const uint32_t MaskBitOrder = (NumBits - 1); + const uint32_t Mask = 1UL << MaskBitOrder; - if ((Piece & (1U << (NumBits-1))) == 0) + if ((Piece & Mask) == 0) return uint64_t(Piece); uint64_t Result = 0; unsigned NextBit = 0; while (true) { - Result |= uint64_t(Piece & ((1U << (NumBits-1))-1)) << NextBit; + Result |= uint64_t(Piece & (Mask - 1)) << NextBit; - if ((Piece & (1U << (NumBits-1))) == 0) + if ((Piece & Mask) == 0) return Result; NextBit += NumBits-1; + if (NextBit >= 64) + return createStringError(std::errc::illegal_byte_sequence, + "Unterminated VBR"); + MaybeRead = Read(NumBits); if (!MaybeRead) return MaybeRead; @@ -299,6 +311,13 @@ public: /// Skip to the end of the file. void skipToEnd() { NextChar = BitcodeBytes.size(); } + + /// Check whether a reservation of Size elements is plausible. + bool isSizePlausible(size_t Size) const { + // Don't allow reserving more elements than the number of bits, assuming + // at least one bit is needed to encode an element. + return Size < BitcodeBytes.size() * 8; + } }; /// When advancing through a bitstream cursor, each advance can discover a few @@ -357,7 +376,7 @@ class BitstreamCursor : SimpleBitstreamCursor { BitstreamBlockInfo *BlockInfo = nullptr; public: - static const size_t MaxChunkSize = sizeof(word_t) * 8; + static const size_t MaxChunkSize = 32; BitstreamCursor() = default; explicit BitstreamCursor(ArrayRef<uint8_t> BitcodeBytes) @@ -521,10 +540,11 @@ private: public: /// Return the abbreviation for the specified AbbrevId. - const BitCodeAbbrev *getAbbrev(unsigned AbbrevID) { + Expected<const BitCodeAbbrev *> getAbbrev(unsigned AbbrevID) { unsigned AbbrevNo = AbbrevID - bitc::FIRST_APPLICATION_ABBREV; if (AbbrevNo >= CurAbbrevs.size()) - report_fatal_error("Invalid abbrev number"); + return createStringError( + std::errc::illegal_byte_sequence, "Invalid abbrev number"); return CurAbbrevs[AbbrevNo].get(); } diff --git a/llvm/include/llvm/Bitstream/BitstreamWriter.h b/llvm/include/llvm/Bitstream/BitstreamWriter.h index 21b260b7b9f3..be6bab5532bd 100644 --- a/llvm/include/llvm/Bitstream/BitstreamWriter.h +++ b/llvm/include/llvm/Bitstream/BitstreamWriter.h @@ -74,16 +74,10 @@ class BitstreamWriter { }; std::vector<BlockInfo> BlockInfoRecords; - void WriteByte(unsigned char Value) { - Out.push_back(Value); - FlushToFile(); - } - void WriteWord(unsigned Value) { Value = support::endian::byte_swap<uint32_t, support::little>(Value); Out.append(reinterpret_cast<const char *>(&Value), reinterpret_cast<const char *>(&Value + 1)); - FlushToFile(); } uint64_t GetNumOfFlushedBytes() const { return FS ? FS->tell() : 0; } @@ -114,7 +108,7 @@ public: /// null, \p O does not flush incrementially, but writes to disk at the end. /// /// \p FlushThreshold is the threshold (unit M) to flush \p O if \p FS is - /// valid. + /// valid. Flushing only occurs at (sub)block boundaries. BitstreamWriter(SmallVectorImpl<char> &O, raw_fd_stream *FS = nullptr, uint32_t FlushThreshold = 512) : Out(O), FS(FS), FlushThreshold(FlushThreshold << 20), CurBit(0), @@ -249,8 +243,8 @@ public: // Emit the bits with VBR encoding, NumBits-1 bits at a time. while (Val >= Threshold) { - Emit(((uint32_t)Val & ((1 << (NumBits-1))-1)) | - (1 << (NumBits-1)), NumBits); + Emit(((uint32_t)Val & ((1 << (NumBits - 1)) - 1)) | (1 << (NumBits - 1)), + NumBits); Val >>= NumBits-1; } @@ -327,6 +321,7 @@ public: CurCodeSize = B.PrevCodeSize; CurAbbrevs = std::move(B.PrevAbbrevs); BlockScope.pop_back(); + FlushToFile(); } //===--------------------------------------------------------------------===// @@ -472,14 +467,12 @@ public: FlushToWord(); // Emit literal bytes. - for (const auto &B : Bytes) { - assert(isUInt<8>(B) && "Value too large to emit as byte"); - WriteByte((unsigned char)B); - } + assert(llvm::all_of(Bytes, [](UIntTy B) { return isUInt<8>(B); })); + Out.append(Bytes.begin(), Bytes.end()); // Align end to 32-bits. while (GetBufferOffset() & 3) - WriteByte(0); + Out.push_back(0); } void emitBlob(StringRef Bytes, bool ShouldEmitSize = true) { emitBlob(makeArrayRef((const uint8_t *)Bytes.data(), Bytes.size()), diff --git a/llvm/include/llvm/CodeGen/AccelTable.h b/llvm/include/llvm/CodeGen/AccelTable.h index 1190d6061e45..c0e976317aef 100644 --- a/llvm/include/llvm/CodeGen/AccelTable.h +++ b/llvm/include/llvm/CodeGen/AccelTable.h @@ -14,19 +14,15 @@ #define LLVM_CODEGEN_ACCELTABLE_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/CodeGen/DIE.h" #include "llvm/CodeGen/DwarfStringPoolEntry.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DJB.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" -#include <cstddef> #include <cstdint> #include <vector> @@ -108,6 +104,8 @@ namespace llvm { class AsmPrinter; class DwarfCompileUnit; class DwarfDebug; +class MCSymbol; +class raw_ostream; /// Interface which the different types of accelerator table data have to /// conform. It serves as a base class for different values of the template diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h index 60442326d6c7..1a09820f80ef 100644 --- a/llvm/include/llvm/CodeGen/Analysis.h +++ b/llvm/include/llvm/CodeGen/Analysis.h @@ -15,14 +15,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Triple.h" #include "llvm/CodeGen/ISDOpcodes.h" -#include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/Support/CodeGen.h" namespace llvm { +template <typename T> class SmallVectorImpl; class GlobalValue; class LLT; class MachineBasicBlock; diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index d911bfd435ae..fb4627c029b0 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -22,9 +22,7 @@ #include "llvm/CodeGen/DwarfStringPoolEntry.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/IR/InlineAsm.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/SourceMgr.h" #include <cstdint> #include <memory> #include <utility> @@ -32,6 +30,7 @@ namespace llvm { +class AddrLabelMap; class BasicBlock; class BlockAddress; class Constant; @@ -176,6 +175,10 @@ private: // function. This is used to calculate the size of the BB section. MCSymbol *CurrentSectionBeginSym = nullptr; + /// This map keeps track of which symbol is being used for the specified basic + /// block's address of label. + std::unique_ptr<AddrLabelMap> AddrLabelSymbols; + // The garbage collection metadata printer table. void *GCMetadataPrinters = nullptr; // Really a DenseMap. @@ -212,6 +215,16 @@ private: /// CFISection type the module needs i.e. either .eh_frame or .debug_frame. CFISection ModuleCFISection = CFISection::None; + /// True if the module contains split-stack functions. This is used to + /// emit .note.GNU-split-stack section as required by the linker for + /// special handling split-stack function calling no-split-stack function. + bool HasSplitStack = false; + + /// True if the module contains no-split-stack functions. This is used to emit + /// .note.GNU-no-split-stack section when it also contains functions without a + /// split stack prologue. + bool HasNoSplitStack = false; + protected: explicit AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer); @@ -254,6 +267,25 @@ public: // given basic block. MCSymbol *getMBBExceptionSym(const MachineBasicBlock &MBB); + /// Return the symbol to be used for the specified basic block when its + /// address is taken. This cannot be its normal LBB label because the block + /// may be accessed outside its containing function. + MCSymbol *getAddrLabelSymbol(const BasicBlock *BB) { + return getAddrLabelSymbolToEmit(BB).front(); + } + + /// Return the symbol to be used for the specified basic block when its + /// address is taken. If other blocks were RAUW'd to this one, we may have + /// to emit them as well, return the whole set. + ArrayRef<MCSymbol *> getAddrLabelSymbolToEmit(const BasicBlock *BB); + + /// If the specified function has had any references to address-taken blocks + /// generated, but the block got deleted, return the symbol now so we can + /// emit it. This prevents emitting a reference to a symbol that has no + /// definition. + void takeDeletedSymbolsForFunction(const Function *F, + std::vector<MCSymbol *> &Result); + /// Return information about object file lowering. const TargetLoweringObjectFile &getObjFileLowering() const; diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h new file mode 100644 index 000000000000..7ae1304cced9 --- /dev/null +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -0,0 +1,109 @@ +//===-- BasicBlockSectionsProfileReader.h - BB sections profile reader pass ==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass creates the basic block cluster info by reading the basic block +// sections profile. The cluster info will be used by the basic-block-sections +// pass to arrange basic blocks in their sections. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H +#define LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +namespace llvm { + +// The cluster information for a machine basic block. +struct BBClusterInfo { + // MachineBasicBlock ID. + unsigned MBBNumber; + // Cluster ID this basic block belongs to. + unsigned ClusterID; + // Position of basic block within the cluster. + unsigned PositionInCluster; +}; + +using ProgramBBClusterInfoMapTy = StringMap<SmallVector<BBClusterInfo>>; + +class BasicBlockSectionsProfileReader : public ImmutablePass { +public: + static char ID; + + BasicBlockSectionsProfileReader(const MemoryBuffer *Buf) + : ImmutablePass(ID), MBuf(Buf) { + initializeBasicBlockSectionsProfileReaderPass( + *PassRegistry::getPassRegistry()); + }; + + BasicBlockSectionsProfileReader() : ImmutablePass(ID) { + initializeBasicBlockSectionsProfileReaderPass( + *PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "Basic Block Sections Profile Reader"; + } + + // Returns true if basic block sections profile exist for function \p + // FuncName. + bool isFunctionHot(StringRef FuncName) const; + + // Returns a pair with first element representing whether basic block sections + // profile exist for the function \p FuncName, and the second element + // representing the basic block sections profile (cluster info) for this + // function. If the first element is true and the second element is empty, it + // means unique basic block sections are desired for all basic blocks of the + // function. + std::pair<bool, SmallVector<BBClusterInfo>> + getBBClusterInfoForFunction(StringRef FuncName) const; + + /// Read profiles of basic blocks if available here. + void initializePass() override; + +private: + StringRef getAliasName(StringRef FuncName) const { + auto R = FuncAliasMap.find(FuncName); + return R == FuncAliasMap.end() ? FuncName : R->second; + } + + // This contains the basic-block-sections profile. + const MemoryBuffer *MBuf = nullptr; + + // This encapsulates the BB cluster information for the whole program. + // + // For every function name, it contains the cluster information for (all or + // some of) its basic blocks. The cluster information for every basic block + // includes its cluster ID along with the position of the basic block in that + // cluster. + ProgramBBClusterInfoMapTy ProgramBBClusterInfo; + + // Some functions have alias names. We use this map to find the main alias + // name for which we have mapping in ProgramBBClusterInfo. + StringMap<StringRef> FuncAliasMap; +}; + +// Creates a BasicBlockSectionsProfileReader pass to parse the basic block +// sections profile. \p Buf is a memory buffer that contains the list of +// functions and basic block ids to selectively enable basic block sections. +ImmutablePass * +createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf); + +} // namespace llvm +#endif // LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 0b2737628923..46be8e030406 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -195,6 +195,10 @@ private: bool VariableMask, bool IsGatherScatter, TTI::TargetCostKind CostKind) { + // We cannot scalarize scalable vectors, so return Invalid. + if (isa<ScalableVectorType>(DataTy)) + return InstructionCost::getInvalid(); + auto *VT = cast<FixedVectorType>(DataTy); // Assume the target does not have support for gather/scatter operations // and provide a rough estimate. @@ -312,6 +316,26 @@ public: return getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace, I); } + unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, + Type *ScalarValTy) const { + auto &&IsSupportedByTarget = [this, ScalarMemTy, ScalarValTy](unsigned VF) { + auto *SrcTy = FixedVectorType::get(ScalarMemTy, VF / 2); + EVT VT = getTLI()->getValueType(DL, SrcTy); + if (getTLI()->isOperationLegal(ISD::STORE, VT) || + getTLI()->isOperationCustom(ISD::STORE, VT)) + return true; + + EVT ValVT = + getTLI()->getValueType(DL, FixedVectorType::get(ScalarValTy, VF / 2)); + EVT LegalizedVT = + getTLI()->getTypeToTransformTo(ScalarMemTy->getContext(), VT); + return getTLI()->isTruncStoreLegal(LegalizedVT, ValVT); + }; + while (VF > 2 && IsSupportedByTarget(VF)) + VF /= 2; + return VF; + } + bool isIndexedLoadLegal(TTI::MemIndexedMode M, Type *Ty, const DataLayout &DL) const { EVT VT = getTLI()->getValueType(DL, Ty); @@ -362,10 +386,9 @@ public: return getTLI()->isTypeLegal(VT); } - InstructionCost getRegUsageForType(Type *Ty) { - InstructionCost Val = getTLI()->getTypeLegalizationCost(DL, Ty).first; - assert(Val >= 0 && "Negative cost!"); - return Val; + unsigned getRegUsageForType(Type *Ty) { + EVT ETy = getTLI()->getValueType(DL, Ty); + return getTLI()->getNumRegisters(Ty->getContext(), ETy); } InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr, @@ -680,6 +703,8 @@ public: bool Insert, bool Extract) { /// FIXME: a bitfield is not a reasonable abstraction for talking about /// which elements are needed from a scalable vector + if (isa<ScalableVectorType>(InTy)) + return InstructionCost::getInvalid(); auto *Ty = cast<FixedVectorType>(InTy); assert(DemandedElts.getBitWidth() == Ty->getNumElements() && @@ -702,6 +727,8 @@ public: /// Helper wrapper for the DemandedElts variant of getScalarizationOverhead. InstructionCost getScalarizationOverhead(VectorType *InTy, bool Insert, bool Extract) { + if (isa<ScalableVectorType>(InTy)) + return InstructionCost::getInvalid(); auto *Ty = cast<FixedVectorType>(InTy); APInt DemandedElts = APInt::getAllOnes(Ty->getNumElements()); @@ -871,7 +898,8 @@ public: InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask, int Index, - VectorType *SubTp) { + VectorType *SubTp, + ArrayRef<const Value *> Args = None) { switch (improveShuffleKindFromMask(Kind, Mask)) { case TTI::SK_Broadcast: @@ -1100,6 +1128,9 @@ public: // TODO: If one of the types get legalized by splitting, handle this // similarly to what getCastInstrCost() does. if (auto *ValVTy = dyn_cast<VectorType>(ValTy)) { + if (isa<ScalableVectorType>(ValTy)) + return InstructionCost::getInvalid(); + unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements(); if (CondTy) CondTy = CondTy->getScalarType(); @@ -1172,11 +1203,12 @@ public: if (CostKind != TTI::TCK_RecipThroughput) return Cost; + const DataLayout &DL = this->getDataLayout(); if (Src->isVectorTy() && // In practice it's not currently possible to have a change in lane // length for extending loads or truncating stores so both types should // have the same scalable property. - TypeSize::isKnownLT(Src->getPrimitiveSizeInBits(), + TypeSize::isKnownLT(DL.getTypeStoreSizeInBits(Src), LT.second.getSizeInBits())) { // This is a vector load that legalizes to a larger type than the vector // itself. Unless the corresponding extending load or truncating store is @@ -1220,6 +1252,11 @@ public: unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond = false, bool UseMaskForGaps = false) { + + // We cannot scalarize scalable vectors, so return Invalid. + if (isa<ScalableVectorType>(VecTy)) + return InstructionCost::getInvalid(); + auto *VT = cast<FixedVectorType>(VecTy); unsigned NumElts = VT->getNumElements(); @@ -1274,8 +1311,7 @@ public: // Scale the cost of the load by the fraction of legal instructions that // will be used. - Cost = divideCeil(UsedInsts.count() * Cost.getValue().getValue(), - NumLegalInsts); + Cost = divideCeil(UsedInsts.count() * *Cost.getValue(), NumLegalInsts); } // Then plus the cost of interleave operation. @@ -1382,6 +1418,26 @@ public: default: break; + case Intrinsic::powi: + if (auto *RHSC = dyn_cast<ConstantInt>(Args[1])) { + bool ShouldOptForSize = I->getParent()->getParent()->hasOptSize(); + if (getTLI()->isBeneficialToExpandPowI(RHSC->getSExtValue(), + ShouldOptForSize)) { + // The cost is modeled on the expansion performed by ExpandPowI in + // SelectionDAGBuilder. + APInt Exponent = RHSC->getValue().abs(); + unsigned ActiveBits = Exponent.getActiveBits(); + unsigned PopCount = Exponent.countPopulation(); + InstructionCost Cost = (ActiveBits + PopCount - 2) * + thisT()->getArithmeticInstrCost( + Instruction::FMul, RetTy, CostKind); + if (RHSC->getSExtValue() < 0) + Cost += thisT()->getArithmeticInstrCost(Instruction::FDiv, RetTy, + CostKind); + return Cost; + } + } + break; case Intrinsic::cttz: // FIXME: If necessary, this should go in target-specific overrides. if (RetVF.isScalar() && getTLI()->isCheapToSpeculateCttz()) @@ -1418,7 +1474,7 @@ public: // The cost of materialising a constant integer vector. return TargetTransformInfo::TCC_Basic; } - case Intrinsic::experimental_vector_extract: { + case Intrinsic::vector_extract: { // FIXME: Handle case where a scalable vector is extracted from a scalable // vector if (isa<ScalableVectorType>(RetTy)) @@ -1428,7 +1484,7 @@ public: cast<VectorType>(Args[0]->getType()), None, Index, cast<VectorType>(RetTy)); } - case Intrinsic::experimental_vector_insert: { + case Intrinsic::vector_insert: { // FIXME: Handle case where a scalable vector is inserted into a scalable // vector if (isa<ScalableVectorType>(Args[1]->getType())) @@ -1471,8 +1527,6 @@ public: } case Intrinsic::fshl: case Intrinsic::fshr: { - if (isa<ScalableVectorType>(RetTy)) - return BaseT::getIntrinsicInstrCost(ICA, CostKind); const Value *X = Args[0]; const Value *Y = Args[1]; const Value *Z = Args[2]; @@ -1512,6 +1566,29 @@ public: } return Cost; } + case Intrinsic::get_active_lane_mask: { + EVT ResVT = getTLI()->getValueType(DL, RetTy, true); + EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true); + + // If we're not expanding the intrinsic then we assume this is cheap + // to implement. + if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgType)) { + std::pair<InstructionCost, MVT> LT = + getTLI()->getTypeLegalizationCost(DL, RetTy); + return LT.first; + } + + // Create the expanded types that will be used to calculate the uadd_sat + // operation. + Type *ExpRetTy = VectorType::get( + ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount()); + IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF); + InstructionCost Cost = + thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind); + Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, RetTy, + CmpInst::ICMP_ULT, CostKind); + return Cost; + } } // Assume that we need to scalarize this intrinsic. @@ -1560,7 +1637,7 @@ public: // Library call cost - other than size, make it expensive. unsigned SingleCallCost = CostKind == TTI::TCK_CodeSize ? 1 : 10; - SmallVector<unsigned, 2> ISDs; + unsigned ISD = 0; switch (IID) { default: { // Scalable vectors cannot be scalarized, so return Invalid. @@ -1605,82 +1682,82 @@ public: // Look for intrinsics that can be lowered directly or turned into a scalar // intrinsic call. case Intrinsic::sqrt: - ISDs.push_back(ISD::FSQRT); + ISD = ISD::FSQRT; break; case Intrinsic::sin: - ISDs.push_back(ISD::FSIN); + ISD = ISD::FSIN; break; case Intrinsic::cos: - ISDs.push_back(ISD::FCOS); + ISD = ISD::FCOS; break; case Intrinsic::exp: - ISDs.push_back(ISD::FEXP); + ISD = ISD::FEXP; break; case Intrinsic::exp2: - ISDs.push_back(ISD::FEXP2); + ISD = ISD::FEXP2; break; case Intrinsic::log: - ISDs.push_back(ISD::FLOG); + ISD = ISD::FLOG; break; case Intrinsic::log10: - ISDs.push_back(ISD::FLOG10); + ISD = ISD::FLOG10; break; case Intrinsic::log2: - ISDs.push_back(ISD::FLOG2); + ISD = ISD::FLOG2; break; case Intrinsic::fabs: - ISDs.push_back(ISD::FABS); + ISD = ISD::FABS; break; case Intrinsic::canonicalize: - ISDs.push_back(ISD::FCANONICALIZE); + ISD = ISD::FCANONICALIZE; break; case Intrinsic::minnum: - ISDs.push_back(ISD::FMINNUM); + ISD = ISD::FMINNUM; break; case Intrinsic::maxnum: - ISDs.push_back(ISD::FMAXNUM); + ISD = ISD::FMAXNUM; break; case Intrinsic::minimum: - ISDs.push_back(ISD::FMINIMUM); + ISD = ISD::FMINIMUM; break; case Intrinsic::maximum: - ISDs.push_back(ISD::FMAXIMUM); + ISD = ISD::FMAXIMUM; break; case Intrinsic::copysign: - ISDs.push_back(ISD::FCOPYSIGN); + ISD = ISD::FCOPYSIGN; break; case Intrinsic::floor: - ISDs.push_back(ISD::FFLOOR); + ISD = ISD::FFLOOR; break; case Intrinsic::ceil: - ISDs.push_back(ISD::FCEIL); + ISD = ISD::FCEIL; break; case Intrinsic::trunc: - ISDs.push_back(ISD::FTRUNC); + ISD = ISD::FTRUNC; break; case Intrinsic::nearbyint: - ISDs.push_back(ISD::FNEARBYINT); + ISD = ISD::FNEARBYINT; break; case Intrinsic::rint: - ISDs.push_back(ISD::FRINT); + ISD = ISD::FRINT; break; case Intrinsic::round: - ISDs.push_back(ISD::FROUND); + ISD = ISD::FROUND; break; case Intrinsic::roundeven: - ISDs.push_back(ISD::FROUNDEVEN); + ISD = ISD::FROUNDEVEN; break; case Intrinsic::pow: - ISDs.push_back(ISD::FPOW); + ISD = ISD::FPOW; break; case Intrinsic::fma: - ISDs.push_back(ISD::FMA); + ISD = ISD::FMA; break; case Intrinsic::fmuladd: - ISDs.push_back(ISD::FMA); + ISD = ISD::FMA; break; case Intrinsic::experimental_constrained_fmuladd: - ISDs.push_back(ISD::STRICT_FMA); + ISD = ISD::STRICT_FMA; break; // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free. case Intrinsic::lifetime_start: @@ -1897,23 +1974,49 @@ public: BinaryOperator::ICmp, MulTy, OverflowTy, CmpInst::ICMP_NE, CostKind); return Cost; } + case Intrinsic::fptosi_sat: + case Intrinsic::fptoui_sat: { + if (Tys.empty()) + break; + Type *FromTy = Tys[0]; + bool IsSigned = IID == Intrinsic::fptosi_sat; + + InstructionCost Cost = 0; + IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FromTy, + {FromTy, FromTy}); + Cost += thisT()->getIntrinsicInstrCost(Attrs1, CostKind); + IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FromTy, + {FromTy, FromTy}); + Cost += thisT()->getIntrinsicInstrCost(Attrs2, CostKind); + Cost += thisT()->getCastInstrCost( + IsSigned ? Instruction::FPToSI : Instruction::FPToUI, RetTy, FromTy, + TTI::CastContextHint::None, CostKind); + if (IsSigned) { + Type *CondTy = RetTy->getWithNewBitWidth(1); + Cost += thisT()->getCmpSelInstrCost( + BinaryOperator::FCmp, FromTy, CondTy, CmpInst::FCMP_UNO, CostKind); + Cost += thisT()->getCmpSelInstrCost( + BinaryOperator::Select, RetTy, CondTy, CmpInst::FCMP_UNO, CostKind); + } + return Cost; + } case Intrinsic::ctpop: - ISDs.push_back(ISD::CTPOP); + ISD = ISD::CTPOP; // In case of legalization use TCC_Expensive. This is cheaper than a // library call but still not a cheap instruction. SingleCallCost = TargetTransformInfo::TCC_Expensive; break; case Intrinsic::ctlz: - ISDs.push_back(ISD::CTLZ); + ISD = ISD::CTLZ; break; case Intrinsic::cttz: - ISDs.push_back(ISD::CTTZ); + ISD = ISD::CTTZ; break; case Intrinsic::bswap: - ISDs.push_back(ISD::BSWAP); + ISD = ISD::BSWAP; break; case Intrinsic::bitreverse: - ISDs.push_back(ISD::BITREVERSE); + ISD = ISD::BITREVERSE; break; } @@ -1921,38 +2024,25 @@ public: std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy); - SmallVector<InstructionCost, 2> LegalCost; - SmallVector<InstructionCost, 2> CustomCost; - for (unsigned ISD : ISDs) { - if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { - if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() && - TLI->isFAbsFree(LT.second)) { - return 0; - } - - // The operation is legal. Assume it costs 1. - // If the type is split to multiple registers, assume that there is some - // overhead to this. - // TODO: Once we have extract/insert subvector cost we need to use them. - if (LT.first > 1) - LegalCost.push_back(LT.first * 2); - else - LegalCost.push_back(LT.first * 1); - } else if (!TLI->isOperationExpand(ISD, LT.second)) { - // If the operation is custom lowered then assume - // that the code is twice as expensive. - CustomCost.push_back(LT.first * 2); + if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { + if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() && + TLI->isFAbsFree(LT.second)) { + return 0; } - } - auto *MinLegalCostI = std::min_element(LegalCost.begin(), LegalCost.end()); - if (MinLegalCostI != LegalCost.end()) - return *MinLegalCostI; - - auto MinCustomCostI = - std::min_element(CustomCost.begin(), CustomCost.end()); - if (MinCustomCostI != CustomCost.end()) - return *MinCustomCostI; + // The operation is legal. Assume it costs 1. + // If the type is split to multiple registers, assume that there is some + // overhead to this. + // TODO: Once we have extract/insert subvector cost we need to use them. + if (LT.first > 1) + return (LT.first * 2); + else + return (LT.first * 1); + } else if (!TLI->isOperationExpand(ISD, LT.second)) { + // If the operation is custom lowered then assume + // that the code is twice as expensive. + return (LT.first * 2); + } // If we can't lower fmuladd into an FMA estimate the cost as a floating // point mul followed by an add. @@ -2061,6 +2151,11 @@ public: /// vector is reduced on each iteration. InstructionCost getTreeReductionCost(unsigned Opcode, VectorType *Ty, TTI::TargetCostKind CostKind) { + // Targets must implement a default value for the scalable case, since + // we don't know how many lanes the vector has. + if (isa<ScalableVectorType>(Ty)) + return InstructionCost::getInvalid(); + Type *ScalarTy = Ty->getElementType(); unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements(); if ((Opcode == Instruction::Or || Opcode == Instruction::And) && @@ -2159,6 +2254,11 @@ public: InstructionCost getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy, bool IsUnsigned, TTI::TargetCostKind CostKind) { + // Targets must implement a default value for the scalable case, since + // we don't know how many lanes the vector has. + if (isa<ScalableVectorType>(Ty)) + return InstructionCost::getInvalid(); + Type *ScalarTy = Ty->getElementType(); Type *ScalarCondTy = CondTy->getElementType(); unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements(); diff --git a/llvm/include/llvm/CodeGen/CFIFixup.h b/llvm/include/llvm/CodeGen/CFIFixup.h new file mode 100644 index 000000000000..40e535106751 --- /dev/null +++ b/llvm/include/llvm/CodeGen/CFIFixup.h @@ -0,0 +1,38 @@ +//===-- CFIFixup.h - Insert CFI remember/restore instructions ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Contains definition of the base CFIFixup pass. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_CFIFIXUP_H +#define LLVM_CODEGEN_CFIFIXUP_H + +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" + +namespace llvm { +class CFIFixup : public MachineFunctionPass { +public: + static char ID; + + CFIFixup() : MachineFunctionPass(ID) { + initializeCFIFixupPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override; +}; +} // namespace llvm + +#endif // LLVM_CODEGEN_CFIFIXUP_H diff --git a/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/llvm/include/llvm/CodeGen/CalcSpillWeights.h index bfd5bab3d1c0..41b7f10cfc38 100644 --- a/llvm/include/llvm/CodeGen/CalcSpillWeights.h +++ b/llvm/include/llvm/CodeGen/CalcSpillWeights.h @@ -9,7 +9,6 @@ #ifndef LLVM_CODEGEN_CALCSPILLWEIGHTS_H #define LLVM_CODEGEN_CALCSPILLWEIGHTS_H -#include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/SlotIndexes.h" namespace llvm { @@ -65,17 +64,6 @@ class VirtRegMap; /// (re)compute li's spill weight and allocation hint. void calculateSpillWeightAndHint(LiveInterval &LI); - /// Compute future expected spill weight of a split artifact of LI - /// that will span between start and end slot indexes. - /// \param LI The live interval to be split. - /// \param Start The expected beginning of the split artifact. Instructions - /// before start will not affect the weight. - /// \param End The expected end of the split artifact. Instructions - /// after end will not affect the weight. - /// \return The expected spill weight of the split artifact. Returns - /// negative weight for unspillable LI. - float futureWeight(LiveInterval &LI, SlotIndex Start, SlotIndex End); - /// Compute spill weights and allocation hints for all virtual register /// live intervals. void calculateSpillWeightsAndHints(); diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h index 8dbcd6b8ab7d..90afbfc32a4e 100644 --- a/llvm/include/llvm/CodeGen/CallingConvLower.h +++ b/llvm/include/llvm/CodeGen/CallingConvLower.h @@ -15,11 +15,9 @@ #define LLVM_CODEGEN_CALLINGCONVLOWER_H #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/Register.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallingConv.h" -#include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Alignment.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h index 270f935b6738..ce278468dffc 100644 --- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h +++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h @@ -19,7 +19,6 @@ namespace llvm { class BasicBlock; -class MachineBasicBlock; /// Encapsulates all of the information needed to generate a stack protector /// check, and signals to isel when initialized that one needs to be generated. /// @@ -213,6 +212,13 @@ private: MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII); +/// Evaluates if the specified FP class test is an inversion of a simpler test. +/// An example is the test "inf|normal|subnormal|zero", which is an inversion +/// of "nan". +/// \param Test The test as specified in 'is_fpclass' intrinsic invocation. +/// \returns The inverted test, or zero, if inversion does not produce simpler +/// test. +unsigned getInvertedFPClassTest(unsigned Test); } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index f6563971f981..f4b1980b9ede 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -15,7 +15,6 @@ #ifndef LLVM_CODEGEN_CODEGENPASSBUILDER_H #define LLVM_CODEGEN_CODEGENPASSBUILDER_H -#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -26,7 +25,6 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/ExpandReductions.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" @@ -35,7 +33,6 @@ #include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" #include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCTargetOptions.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" @@ -43,7 +40,6 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" @@ -51,7 +47,6 @@ #include "llvm/Transforms/Scalar/MergeICmps.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" -#include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include <cassert> @@ -668,6 +663,10 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const { // Expand reduction intrinsics into shuffle sequences if the target wants to. addPass(ExpandReductionsPass()); + + // Convert conditional moves to conditional jumps when profitable. + if (getOptLevel() != CodeGenOpt::None && !Opt.DisableSelectOptimize) + addPass(SelectOptimizePass()); } /// Turn exception handling constructs into something the code generators can @@ -751,7 +750,7 @@ template <typename Derived> Error CodeGenPassBuilder<Derived>::addCoreISelPasses( AddMachinePass &addPass) const { // Enable FastISel with -fast-isel, but allow that to be overridden. - TM.setO0WantsFastISel(Opt.EnableFastISelOption.getValueOr(true)); + TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true)); // Determine an instruction selector. enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index 73d39fecc268..9281ed723854 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -16,11 +16,6 @@ #define LLVM_CODEGEN_COMMANDFLAGS_H #include "llvm/ADT/FloatingPointMode.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" #include "llvm/Support/CodeGen.h" #include "llvm/Target/TargetOptions.h" #include <string> @@ -29,6 +24,9 @@ namespace llvm { class Module; +class AttrBuilder; +class Function; +class Triple; namespace codegen { @@ -62,6 +60,8 @@ bool getEnableNoNaNsFPMath(); bool getEnableNoSignedZerosFPMath(); +bool getEnableApproxFuncFPMath(); + bool getEnableNoTrappingFPMath(); DenormalMode::DenormalModeKind getDenormalFPMath(); @@ -93,6 +93,8 @@ std::string getTrapFuncName(); bool getUseCtors(); +bool getLowerGlobalDtorsViaCxaAtExit(); + bool getRelaxELFRelocations(); bool getDataSections(); @@ -140,6 +142,8 @@ bool getDebugStrictDwarf(); unsigned getAlignLoops(); +bool getJMCInstrument(); + /// Create this object with static storage to register codegen-related command /// line options. struct RegisterCodeGenFlags { diff --git a/llvm/include/llvm/CodeGen/DFAPacketizer.h b/llvm/include/llvm/CodeGen/DFAPacketizer.h index 9cdaedc9e861..aba6503a6a1f 100644 --- a/llvm/include/llvm/CodeGen/DFAPacketizer.h +++ b/llvm/include/llvm/CodeGen/DFAPacketizer.h @@ -25,9 +25,7 @@ #ifndef LLVM_CODEGEN_DFAPACKETIZER_H #define LLVM_CODEGEN_DFAPACKETIZER_H -#include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/ScheduleDAGMutation.h" #include "llvm/Support/Automaton.h" #include <cstdint> #include <map> @@ -38,6 +36,7 @@ namespace llvm { class DefaultVLIWScheduler; +class ScheduleDAGMutation; class InstrItineraryData; class MachineFunction; class MachineInstr; diff --git a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h index 2ac9d938d281..465829159e42 100644 --- a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h +++ b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h @@ -12,12 +12,12 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/LexicalScopes.h" #include <utility> namespace llvm { class DILocation; +class LexicalScopes; class DINode; class MachineFunction; class MachineInstr; diff --git a/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h b/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h index abeba62707c1..f19d321793e9 100644 --- a/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h +++ b/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h @@ -9,7 +9,7 @@ #ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H #define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H -#include "llvm/ADT/PointerIntPair.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/StringMap.h" namespace llvm { @@ -20,49 +20,91 @@ class MCSymbol; struct DwarfStringPoolEntry { static constexpr unsigned NotIndexed = -1; - MCSymbol *Symbol; - uint64_t Offset; - unsigned Index; + MCSymbol *Symbol = nullptr; + uint64_t Offset = 0; + unsigned Index = 0; bool isIndexed() const { return Index != NotIndexed; } }; -/// String pool entry reference. +/// DwarfStringPoolEntryRef: Dwarf string pool entry reference. +/// +/// Dwarf string pool entry keeps string value and its data. +/// There are two variants how data are represented: +/// +/// 1. By value - StringMapEntry<DwarfStringPoolEntry>. +/// 2. By pointer - StringMapEntry<DwarfStringPoolEntry *>. +/// +/// The "By pointer" variant allows for reducing memory usage for the case +/// when string pool entry does not have data: it keeps the null pointer +/// and so no need to waste space for the full DwarfStringPoolEntry. +/// It is recommended to use "By pointer" variant if not all entries +/// of dwarf string pool have corresponding DwarfStringPoolEntry. + class DwarfStringPoolEntryRef { - PointerIntPair<const StringMapEntry<DwarfStringPoolEntry> *, 1, bool> - MapEntryAndIndexed; + /// Pointer type for "By value" string entry. + using ByValStringEntryPtr = const StringMapEntry<DwarfStringPoolEntry> *; - const StringMapEntry<DwarfStringPoolEntry> *getMapEntry() const { - return MapEntryAndIndexed.getPointer(); - } + /// Pointer type for "By pointer" string entry. + using ByPtrStringEntryPtr = const StringMapEntry<DwarfStringPoolEntry *> *; + + /// Pointer to the dwarf string pool Entry. + PointerUnion<ByValStringEntryPtr, ByPtrStringEntryPtr> MapEntry = nullptr; public: DwarfStringPoolEntryRef() = default; - DwarfStringPoolEntryRef(const StringMapEntry<DwarfStringPoolEntry> &Entry, - bool Indexed) - : MapEntryAndIndexed(&Entry, Indexed) {} - explicit operator bool() const { return getMapEntry(); } + /// ASSUMPTION: DwarfStringPoolEntryRef keeps pointer to \p Entry, + /// thus specified entry mustn`t be reallocated. + DwarfStringPoolEntryRef(const StringMapEntry<DwarfStringPoolEntry> &Entry) + : MapEntry(&Entry) {} + + /// ASSUMPTION: DwarfStringPoolEntryRef keeps pointer to \p Entry, + /// thus specified entry mustn`t be reallocated. + DwarfStringPoolEntryRef(const StringMapEntry<DwarfStringPoolEntry *> &Entry) + : MapEntry(&Entry) { + assert(MapEntry.get<ByPtrStringEntryPtr>()->second != nullptr); + } + + explicit operator bool() const { return !MapEntry.isNull(); } + + /// \returns symbol for the dwarf string. MCSymbol *getSymbol() const { - assert(getMapEntry()->second.Symbol && "No symbol available!"); - return getMapEntry()->second.Symbol; + assert(getEntry().Symbol && "No symbol available!"); + return getEntry().Symbol; } - uint64_t getOffset() const { return getMapEntry()->second.Offset; } - bool isIndexed() const { return MapEntryAndIndexed.getInt(); } + + /// \returns offset for the dwarf string. + uint64_t getOffset() const { return getEntry().Offset; } + + /// \returns index for the dwarf string. unsigned getIndex() const { - assert(isIndexed()); - assert(getMapEntry()->getValue().isIndexed()); - return getMapEntry()->second.Index; + assert(getEntry().isIndexed() && "Index is not set!"); + return getEntry().Index; + } + + /// \returns string. + StringRef getString() const { + if (MapEntry.is<ByValStringEntryPtr>()) + return MapEntry.get<ByValStringEntryPtr>()->first(); + + return MapEntry.get<ByPtrStringEntryPtr>()->first(); + } + + /// \returns the entire string pool entry for convenience. + const DwarfStringPoolEntry &getEntry() const { + if (MapEntry.is<ByValStringEntryPtr>()) + return MapEntry.get<ByValStringEntryPtr>()->second; + + return *MapEntry.get<ByPtrStringEntryPtr>()->second; } - StringRef getString() const { return getMapEntry()->first(); } - /// Return the entire string pool entry for convenience. - DwarfStringPoolEntry getEntry() const { return getMapEntry()->getValue(); } bool operator==(const DwarfStringPoolEntryRef &X) const { - return getMapEntry() == X.getMapEntry(); + return MapEntry.getOpaqueValue() == X.MapEntry.getOpaqueValue(); } + bool operator!=(const DwarfStringPoolEntryRef &X) const { - return getMapEntry() != X.getMapEntry(); + return MapEntry.getOpaqueValue() != X.MapEntry.getOpaqueValue(); } }; diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h index 775698a66ada..8be97d2c2095 100644 --- a/llvm/include/llvm/CodeGen/FastISel.h +++ b/llvm/include/llvm/CodeGen/FastISel.h @@ -24,15 +24,15 @@ #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/MachineValueType.h" -#include <algorithm> #include <cstdint> #include <utility> namespace llvm { class AllocaInst; +class Instruction; +class IntrinsicInst; class BasicBlock; class CallInst; class Constant; @@ -212,6 +212,7 @@ protected: const TargetRegisterInfo &TRI; const TargetLibraryInfo *LibInfo; bool SkipTargetIndependentISel; + bool UseInstrRefDebugInfo = false; /// The position of the last instruction for materializing constants /// for use in the current block. It resets to EmitStartPt when it makes sense @@ -318,6 +319,12 @@ public: /// Reset InsertPt to the given old insert position. void leaveLocalValueArea(SavePoint Old); + /// Signal whether instruction referencing variable locations are desired for + /// this function's debug-info. + void useInstrRefDebugInfo(bool Flag) { + UseInstrRefDebugInfo = Flag; + } + protected: explicit FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, diff --git a/llvm/include/llvm/CodeGen/FaultMaps.h b/llvm/include/llvm/CodeGen/FaultMaps.h index 8a8b1d2e6008..c228bb895edd 100644 --- a/llvm/include/llvm/CodeGen/FaultMaps.h +++ b/llvm/include/llvm/CodeGen/FaultMaps.h @@ -10,7 +10,6 @@ #define LLVM_CODEGEN_FAULTMAPS_H #include "llvm/MC/MCSymbol.h" -#include "llvm/Support/Endian.h" #include <map> #include <vector> diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index 524730d53694..f8156ce73196 100644 --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -101,6 +101,10 @@ public: // Value was lowered to tied def and gc.relocate should be replaced with // copy from vreg. VReg, + // Value was lowered to tied def and gc.relocate should be replaced with + // SDValue kept in StatepointLoweringInfo structure. This valid for local + // relocates only. + SDValueNode, } type = NoRelocate; // Payload contains either frame index of the stack slot in which the value // was spilled, or virtual register which contains the re-definition. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h index 4f95335db74b..4d9694347f17 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -13,10 +13,10 @@ #define LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" namespace llvm { +class GISelInstProfileBuilder; /// Defines a builder that does CSE of MachineInstructions using GISelCSEInfo. /// Eg usage. /// diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index f9663fadb868..9bf1c134618c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -17,25 +17,26 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" -#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetCallingConv.h" -#include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/LowLevelTypeImpl.h" #include "llvm/Support/MachineValueType.h" #include <cstdint> #include <functional> namespace llvm { +class AttributeList; class CallBase; class DataLayout; class Function; class FunctionLoweringInfo; class MachineIRBuilder; +class MachineFunction; struct MachinePointerInfo; class MachineRegisterInfo; class TargetLowering; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h index 795686980842..8c295428afe8 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -15,7 +15,6 @@ #define LLVM_CODEGEN_GLOBALISEL_COMBINER_H #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { class MachineRegisterInfo; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 45c27c25aea0..73edc3c37970 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -17,16 +17,20 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINERHELPER_H #define LLVM_CODEGEN_GLOBALISEL_COMBINERHELPER_H -#include "llvm/ADT/APFloat.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" -#include "llvm/CodeGen/LowLevelType.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/Register.h" -#include "llvm/Support/Alignment.h" +#include "llvm/Support/LowLevelTypeImpl.h" +#include <functional> namespace llvm { class GISelChangeObserver; +class APFloat; +class APInt; +class GPtrAdd; +class GStore; +class GZExtLoad; class MachineIRBuilder; class MachineInstrBuilder; class MachineRegisterInfo; @@ -124,10 +128,20 @@ public: const TargetLowering &getTargetLowering() const; + /// \returns true if the combiner is running pre-legalization. + bool isPreLegalize() const; + + /// \returns true if \p Query is legal on the target. + bool isLegal(const LegalityQuery &Query) const; + /// \return true if the combine is running prior to legalization, or if \p /// Query is legal on the target. bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const; + /// \return true if the combine is running prior to legalization, or if \p Ty + /// is a legal integer constant type on the target. + bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const; + /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const; @@ -529,6 +543,13 @@ public: /// Combine G_UREM x, (known power of 2) to an add and bitmasking. void applySimplifyURemByPow2(MachineInstr &MI); + /// Push a binary operator through a select on constants. + /// + /// binop (select cond, K0, K1), K2 -> + /// select cond, (binop K0, K2), (binop K1, K2) + bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo); + bool applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo); + bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo); @@ -645,6 +666,14 @@ public: /// (G_SMULO x, 2) -> (G_SADDO x, x) bool matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo); + /// Match: + /// (G_*MULO x, 0) -> 0 + no carry out + bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo); + + /// Match: + /// (G_*ADDO x, 0) -> x + no carry out + bool matchAddOBy0(MachineInstr &MI, BuildFnTy &MatchInfo); + /// Transform (fadd x, fneg(y)) -> (fsub x, y) /// (fadd fneg(x), y) -> (fsub y, x) /// (fsub x, fneg(y)) -> (fadd x, y) @@ -702,6 +731,15 @@ public: bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo); + /// Fold boolean selects to logical operations. + bool matchSelectToLogical(MachineInstr &MI, BuildFnTy &MatchInfo); + + bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info); + + /// Transform G_ADD(x, G_SUB(y, x)) to y. + /// Transform G_ADD(G_SUB(y, x), x) to y. + bool matchAddSubSameReg(MachineInstr &MI, Register &Src); + private: /// Given a non-indexed load or store instruction \p MI, find an offset that /// can be usefully and legally folded into it as a post-indexing operation. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 7d198fada411..3ec6a1da201e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -28,7 +28,7 @@ class GISelWorkList { SmallVector<MachineInstr *, N> Worklist; DenseMap<MachineInstr *, unsigned> WorklistMap; -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS bool Finalized = true; #endif @@ -49,7 +49,7 @@ public: // of most passes. void deferred_insert(MachineInstr *I) { Worklist.push_back(I); -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS Finalized = false; #endif } @@ -65,21 +65,25 @@ public: for (unsigned i = 0; i < Worklist.size(); ++i) if (!WorklistMap.try_emplace(Worklist[i], i).second) llvm_unreachable("Duplicate elements in the list"); -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS Finalized = true; #endif } /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS assert(Finalized && "GISelWorkList used without finalizing"); +#endif if (WorklistMap.try_emplace(I, Worklist.size()).second) Worklist.push_back(I); } /// Remove I from the worklist if it exists. void remove(const MachineInstr *I) { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS assert((Finalized || WorklistMap.empty()) && "Neither finalized nor empty"); +#endif auto It = WorklistMap.find(I); if (It == WorklistMap.end()) return; // Not in worklist. @@ -96,7 +100,9 @@ public: } MachineInstr *pop_back_val() { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS assert(Finalized && "GISelWorkList used without finalizing"); +#endif MachineInstr *I; do { I = Worklist.pop_back_val(); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h index 7103656365b1..58fe48200e73 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H #define LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H +#include "llvm/IR/Instructions.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" @@ -226,6 +227,37 @@ public: } }; +/// Represent a G_ICMP or G_FCMP. +class GAnyCmp : public GenericMachineInstr { +public: + CmpInst::Predicate getCond() const { + return static_cast<CmpInst::Predicate>(getOperand(1).getPredicate()); + } + Register getLHSReg() const { return getReg(2); } + Register getRHSReg() const { return getReg(3); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_ICMP || + MI->getOpcode() == TargetOpcode::G_FCMP; + } +}; + +/// Represent a G_ICMP. +class GICmp : public GAnyCmp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_ICMP; + } +}; + +/// Represent a G_FCMP. +class GFCmp : public GAnyCmp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_FCMP; + } +}; + } // namespace llvm #endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index ebe16cd4f58c..5e7428a5edc5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -22,11 +22,10 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CodeGenCommonISel.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/SwitchLoweringUtils.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/CodeGen.h" #include <memory> @@ -248,12 +247,6 @@ private: bool translateInlineAsm(const CallBase &CB, MachineIRBuilder &MIRBuilder); - /// Returns true if the value should be split into multiple LLTs. - /// If \p Offsets is given then the split type's offsets will be stored in it. - /// If \p Offsets is not empty it will be cleared first. - bool valueIsSplit(const Value &V, - SmallVectorImpl<uint64_t> *Offsets = nullptr); - /// Common code for translating normal calls or invokes. bool translateCallBase(const CallBase &CB, MachineIRBuilder &MIRBuilder); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h index 4a72621ec61e..60c7694725a5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h @@ -13,8 +13,10 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECT_H #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECT_H -#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Support/CodeGen.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 03f4f3bf0b19..8ea45e576e4d 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -18,12 +18,9 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/GlobalISel/Utils.h" -#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/Support/CodeGenCoverage.h" +#include "llvm/IR/Function.h" #include "llvm/Support/LowLevelTypeImpl.h" #include <bitset> #include <cstddef> @@ -34,6 +31,10 @@ namespace llvm { +class BlockFrequencyInfo; +class CodeGenCoverage; +class MachineBasicBlock; +class ProfileSummaryInfo; class APInt; class APFloat; class GISelKnownBits; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index bc9f952146c2..c06b33d11170 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -17,16 +17,17 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/Support/CodeGenCoverage.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -673,7 +674,7 @@ bool InstructionSelector::executeMatchTable( ComplexRendererFns Renderer = (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( State.MIs[InsnID]->getOperand(OpIdx)); - if (Renderer.hasValue()) + if (Renderer) State.Renderers[RendererID] = Renderer.getValue(); else if (handleReject() == RejectAndGiveUp) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 38d2fe28063a..6802591b6350 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -24,10 +24,10 @@ #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Register.h" +#include "llvm/IR/Constants.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "legalizer" -using namespace llvm::MIPatternMatch; namespace llvm { class LegalizationArtifactCombiner { @@ -56,6 +56,7 @@ public: SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + using namespace llvm::MIPatternMatch; assert(MI.getOpcode() == TargetOpcode::G_ANYEXT); Builder.setInstrAndDebugLoc(MI); @@ -109,6 +110,7 @@ public: SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + using namespace llvm::MIPatternMatch; assert(MI.getOpcode() == TargetOpcode::G_ZEXT); Builder.setInstrAndDebugLoc(MI); @@ -170,6 +172,7 @@ public: bool tryCombineSExt(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs) { + using namespace llvm::MIPatternMatch; assert(MI.getOpcode() == TargetOpcode::G_SEXT); Builder.setInstrAndDebugLoc(MI); @@ -227,6 +230,7 @@ public: SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + using namespace llvm::MIPatternMatch; assert(MI.getOpcode() == TargetOpcode::G_TRUNC); Builder.setInstr(MI); @@ -1281,6 +1285,8 @@ private: /// Looks through copy instructions and returns the actual /// source register. Register lookThroughCopyInstrs(Register Reg) { + using namespace llvm::MIPatternMatch; + Register TmpReg; while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { if (MRI.getType(TmpReg).isValid()) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h index c19f1d5330ba..7884b3f2ea6e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -20,11 +20,17 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZER_H #define LLVM_CODEGEN_GLOBALISEL_LEGALIZER_H -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { +class LegalizerInfo; +class MachineIRBuilder; +class MachineInstr; +class GISelChangeObserver; class LostDebugLocObserver; class Legalizer : public MachineFunctionPass { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 3b2f937375eb..c6c57ac07f0e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -21,14 +21,22 @@ #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H #include "llvm/CodeGen/GlobalISel/CallLowering.h" -#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/LowLevelType.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/CodeGen/TargetOpcodes.h" namespace llvm { // Forward declarations. +class APInt; +class GAnyLoad; +class GLoadStore; +class GStore; +class GenericMachineInstr; +class MachineFunction; +class MachineIRBuilder; +class MachineInstr; +class MachineInstrBuilder; +struct MachinePointerInfo; +template <typename T> class SmallVectorImpl; class LegalizerInfo; class MachineRegisterInfo; class GISelChangeObserver; @@ -159,10 +167,6 @@ public: /// def by inserting a G_BITCAST from \p CastTy void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); - /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with - /// G_IMPLICIT_DEF, and producing dead results. - Register widenWithUnmerge(LLT WideTy, Register OrigReg); - private: LegalizeResult widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 17cb53dd2d5b..c0cad8ff675d 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -14,26 +14,26 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/LowLevelTypeImpl.h" -#include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> #include <tuple> -#include <unordered_map> #include <utility> namespace llvm { extern cl::opt<bool> DisableGISelLegalityCheck; +class MachineFunction; +class raw_ostream; class LegalizerHelper; class MachineInstr; class MachineRegisterInfo; @@ -327,8 +327,14 @@ LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); /// index. LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); -/// True iff the specified MMO index has a size that is not a power of 2 +/// True iff the specified MMO index has a size (rounded to bytes) that is not a +/// power of 2. LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); + +/// True iff the specified MMO index has a size that is not an even byte size, +/// or that even byte size is not a power of 2. +LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx); + /// True iff the specified type index is a vector whose element count is not a /// power of 2. LegalityPredicate numElementsNotPow2(unsigned TypeIdx); @@ -351,6 +357,14 @@ LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); /// Keep the same scalar or element type as the given type. LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); +/// Keep the same scalar or element type as \p TypeIdx, but take the number of +/// elements from \p FromTypeIdx. +LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx); + +/// Keep the same scalar or element type as \p TypeIdx, but take the number of +/// elements from \p Ty. +LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty); + /// Change the scalar size or element size to have the same scalar size as type /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and /// only changes the size. @@ -800,11 +814,23 @@ public: return actionIf(LegalizeAction::Unsupported, LegalityPredicates::memSizeInBytesNotPow2(0)); } + + /// Lower a memory operation if the memory size, rounded to bytes, is not a + /// power of 2. For example, this will not trigger for s1 or s7, but will for + /// s24. LegalizeRuleSet &lowerIfMemSizeNotPow2() { return actionIf(LegalizeAction::Lower, LegalityPredicates::memSizeInBytesNotPow2(0)); } + /// Lower a memory operation if the memory access size is not a round power of + /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely + /// what you want (e.g. this will lower s1, s7 and s24). + LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() { + return actionIf(LegalizeAction::Lower, + LegalityPredicates::memSizeNotByteSizePow2(0)); + } + LegalizeRuleSet &customIf(LegalityPredicate Predicate) { // We have no choice but conservatively assume that a custom action with a // free-form user provided Predicate properly handles all type indices: diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h index 0845c001abdb..6efe7c7c9bbd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h @@ -17,18 +17,19 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" -#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" -#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" namespace llvm { // Forward declarations. +class AnalysisUsage; +class GStore; +class LegalizerInfo; +class MachineBasicBlock; +class MachineInstr; +class TargetLowering; +struct LegalityQuery; class MachineRegisterInfo; namespace GISelAddressing { /// Helper struct to store a base, index and offset that forms an address diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h index 1d1afff7f934..9ea0d095eeb1 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h @@ -22,11 +22,14 @@ #define LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H #include "llvm/ADT/SetVector.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { // Forward declarations. +class AnalysisUsage; +class MachineBasicBlock; +class MachineInstr; +class MachineOperand; class MachineRegisterInfo; class TargetTransformInfo; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index daf1ff052983..1cacf96620f0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -94,6 +94,48 @@ inline ConstantMatch<int64_t> m_ICst(int64_t &Cst) { return ConstantMatch<int64_t>(Cst); } +template <typename ConstT> +inline Optional<ConstT> matchConstantSplat(Register, + const MachineRegisterInfo &); + +template <> +inline Optional<APInt> matchConstantSplat(Register Reg, + const MachineRegisterInfo &MRI) { + return getIConstantSplatVal(Reg, MRI); +} + +template <> +inline Optional<int64_t> matchConstantSplat(Register Reg, + const MachineRegisterInfo &MRI) { + return getIConstantSplatSExtVal(Reg, MRI); +} + +template <typename ConstT> struct ICstOrSplatMatch { + ConstT &CR; + ICstOrSplatMatch(ConstT &C) : CR(C) {} + bool match(const MachineRegisterInfo &MRI, Register Reg) { + if (auto MaybeCst = matchConstant<ConstT>(Reg, MRI)) { + CR = *MaybeCst; + return true; + } + + if (auto MaybeCstSplat = matchConstantSplat<ConstT>(Reg, MRI)) { + CR = *MaybeCstSplat; + return true; + } + + return false; + }; +}; + +inline ICstOrSplatMatch<APInt> m_ICstOrSplat(APInt &Cst) { + return ICstOrSplatMatch<APInt>(Cst); +} + +inline ICstOrSplatMatch<int64_t> m_ICstOrSplat(int64_t &Cst) { + return ICstOrSplatMatch<int64_t>(Cst); +} + struct GCstAndRegMatch { Optional<ValueAndVReg> &ValReg; GCstAndRegMatch(Optional<ValueAndVReg> &ValReg) : ValReg(ValReg) {} diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index c4c2fc076dd8..16ba568c1be9 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -13,19 +13,26 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H #define LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H -#include "llvm/CodeGen/GlobalISel/CSEInfo.h" -#include "llvm/CodeGen/LowLevelType.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Module.h" namespace llvm { // Forward declarations. +class APInt; +class BlockAddress; +class Constant; +class ConstantFP; +class ConstantInt; +class DataLayout; +class GISelCSEInfo; +class GlobalValue; +class TargetRegisterClass; class MachineFunction; class MachineInstr; class TargetInstrInfo; @@ -942,22 +949,6 @@ public: /// Build and insert \p Res = IMPLICIT_DEF. MachineInstrBuilder buildUndef(const DstOp &Res); - /// Build and insert instructions to put \p Ops together at the specified p - /// Indices to form a larger register. - /// - /// If the types of the input registers are uniform and cover the entirity of - /// \p Res then a G_MERGE_VALUES will be produced. Otherwise an IMPLICIT_DEF - /// followed by a sequence of G_INSERT instructions. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre The final element of the sequence must not extend past the end of the - /// destination register. - /// \pre The bits defined by each Op (derived from index and scalar size) must - /// not overlap. - /// \pre \p Indices must be in ascending order of bit position. - void buildSequence(Register Res, ArrayRef<Register> Ops, - ArrayRef<uint64_t> Indices); - /// Build and insert \p Res = G_MERGE_VALUES \p Op0, ... /// /// G_MERGE_VALUES combines the input elements contiguously into a larger @@ -1001,6 +992,11 @@ public: MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef<Register> Ops); + /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... where each OpN is + /// built with G_CONSTANT. + MachineInstrBuilder buildBuildVectorConstant(const DstOp &Res, + ArrayRef<APInt> Ops); + /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill /// the number of elements MachineInstrBuilder buildSplatVector(const DstOp &Res, @@ -1442,8 +1438,8 @@ public: /// Build and insert \p Res = G_SUB \p Op0, \p Op1 /// - /// G_SUB sets \p Res to the sum of integer parameters \p Op0 and \p Op1, - /// truncated to their width. + /// G_SUB sets \p Res to the difference of integer parameters \p Op0 and + /// \p Op1, truncated to their width. /// /// \pre setBasicBlock or setMI must have been called. /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers @@ -1459,7 +1455,7 @@ public: /// Build and insert \p Res = G_MUL \p Op0, \p Op1 /// - /// G_MUL sets \p Res to the sum of integer parameters \p Op0 and \p Op1, + /// G_MUL sets \p Res to the product of integer parameters \p Op0 and \p Op1, /// truncated to their width. /// /// \pre setBasicBlock or setMI must have been called. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index 45006eecfce6..d0918485249d 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -66,10 +66,10 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include <cassert> #include <cstdint> #include <memory> diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index aed915d2cc4b..78f1b49da822 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -15,18 +15,20 @@ #define LLVM_CODEGEN_GLOBALISEL_UTILS_H #include "GISelWorkList.h" -#include "LostDebugLocObserver.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/StringRef.h" -#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/Register.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/Support/Alignment.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/LowLevelTypeImpl.h" #include <cstdint> namespace llvm { class AnalysisUsage; +class LostDebugLocObserver; +class MachineBasicBlock; class BlockFrequencyInfo; class GISelKnownBits; class MachineFunction; @@ -267,13 +269,10 @@ Optional<APFloat> ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const MachineRegisterInfo &MRI); /// Tries to constant fold a vector binop with sources \p Op1 and \p Op2. -/// If successful, returns the G_BUILD_VECTOR representing the folded vector -/// constant. \p MIB should have an insertion point already set to create new -/// G_CONSTANT instructions as needed. -Register ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, - const Register Op2, - const MachineRegisterInfo &MRI, - MachineIRBuilder &MIB); +/// Returns an empty vector on failure. +SmallVector<APInt> ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, + const Register Op2, + const MachineRegisterInfo &MRI); Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const Register Op1, uint64_t Imm, const MachineRegisterInfo &MRI); @@ -374,9 +373,23 @@ public: /// If \p MI is not a splat, returns None. Optional<int> getSplatIndex(MachineInstr &MI); -/// Returns a scalar constant of a G_BUILD_VECTOR splat if it exists. -Optional<int64_t> getBuildVectorConstantSplat(const MachineInstr &MI, - const MachineRegisterInfo &MRI); +/// \returns the scalar integral splat value of \p Reg if possible. +Optional<APInt> getIConstantSplatVal(const Register Reg, + const MachineRegisterInfo &MRI); + +/// \returns the scalar integral splat value defined by \p MI if possible. +Optional<APInt> getIConstantSplatVal(const MachineInstr &MI, + const MachineRegisterInfo &MRI); + +/// \returns the scalar sign extended integral splat value of \p Reg if +/// possible. +Optional<int64_t> getIConstantSplatSExtVal(const Register Reg, + const MachineRegisterInfo &MRI); + +/// \returns the scalar sign extended integral splat value defined by \p MI if +/// possible. +Optional<int64_t> getIConstantSplatSExtVal(const MachineInstr &MI, + const MachineRegisterInfo &MRI); /// Returns a floating point scalar constant of a build vector splat if it /// exists. When \p AllowUndef == true some elements can be undef but not all. @@ -408,6 +421,30 @@ bool isBuildVectorAllOnes(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef = false); +/// Return true if the specified instruction is known to be a constant, or a +/// vector of constants. +/// +/// If \p AllowFP is true, this will consider G_FCONSTANT in addition to +/// G_CONSTANT. If \p AllowOpaqueConstants is true, constant-like instructions +/// such as G_GLOBAL_VALUE will also be considered. +bool isConstantOrConstantVector(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + bool AllowFP = true, + bool AllowOpaqueConstants = true); + +/// Return true if the value is a constant 0 integer or a splatted vector of a +/// constant 0 integer (with no undefs if \p AllowUndefs is false). This will +/// handle G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC as truncation is not an issue +/// for null values. +bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, + bool AllowUndefs = false); + +/// Return true if the value is a constant -1 integer or a splatted vector of a +/// constant -1 integer (with no undefs if \p AllowUndefs is false). +bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + bool AllowUndefs = false); + /// \returns a value when \p MI is a vector splat. The splat can be either a /// Register or a constant. /// diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index b07c7cd3db3a..120f89952a95 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -281,12 +281,25 @@ enum NodeType { /// Carry-using nodes for multiple precision addition and subtraction. /// These nodes take three operands: The first two are the normal lhs and - /// rhs to the add or sub, and the third is a boolean indicating if there - /// is an incoming carry. These nodes produce two results: the normal - /// result of the add or sub, and the output carry so they can be chained - /// together. The use of this opcode is preferable to adde/sube if the - /// target supports it, as the carry is a regular value rather than a - /// glue, which allows further optimisation. + /// rhs to the add or sub, and the third is a boolean value that is 1 if and + /// only if there is an incoming carry/borrow. These nodes produce two + /// results: the normal result of the add or sub, and a boolean value that is + /// 1 if and only if there is an outgoing carry/borrow. + /// + /// Care must be taken if these opcodes are lowered to hardware instructions + /// that use the inverse logic -- 0 if and only if there is an + /// incoming/outgoing carry/borrow. In such cases, you must preserve the + /// semantics of these opcodes by inverting the incoming carry/borrow, feeding + /// it to the add/sub hardware instruction, and then inverting the outgoing + /// carry/borrow. + /// + /// The use of these opcodes is preferable to adde/sube if the target supports + /// it, as the carry is a regular value rather than a glue, which allows + /// further optimisation. + /// + /// These opcodes are different from [US]{ADD,SUB}O in that ADDCARRY/SUBCARRY + /// consume and produce a carry/borrow, whereas [US]{ADD,SUB}O produce an + /// overflow. ADDCARRY, SUBCARRY, @@ -294,7 +307,7 @@ enum NodeType { /// subtraction. These nodes take three operands: The first two are normal lhs /// and rhs to the add or sub, and the third is a boolean indicating if there /// is an incoming carry. They produce two results: the normal result of the - /// add or sub, and a boolean that indicates if an overflow occured (*not* + /// add or sub, and a boolean that indicates if an overflow occurred (*not* /// flag, because it may be a store to memory, etc.). If the type of the /// boolean is not i1 then the high bits conform to getBooleanContents. SADDO_CARRY, @@ -462,6 +475,9 @@ enum NodeType { STRICT_FSETCC, STRICT_FSETCCS, + // FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic. + FPTRUNC_ROUND, + /// FMA - Perform a * b + c with no intermediate rounding step. FMA, @@ -482,6 +498,13 @@ enum NodeType { /// Returns platform specific canonical encoding of a floating point number. FCANONICALIZE, + /// Performs a check of floating point class property, defined by IEEE-754. + /// The first operand is the floating point value to check. The second operand + /// specifies the checked property and is a TargetConstant which specifies + /// test in the same way as intrinsic 'is_fpclass'. + /// Returns boolean value. + IS_FPCLASS, + /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector /// with the specified, possibly variable, elements. The types of the /// operands must match the vector element type, except that integer types @@ -614,6 +637,17 @@ enum NodeType { MULHU, MULHS, + /// AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of + /// type i[N+1], halving the result by shifting it one bit right. + /// shr(add(ext(X), ext(Y)), 1) + AVGFLOORS, + AVGFLOORU, + /// AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an + /// integer of type i[N+2], add 1 and halve the result by shifting it one bit + /// right. shr(add(ext(X), ext(Y), 1), 1) + AVGCEILS, + AVGCEILU, + // ABDS/ABDU - Absolute difference - Return the absolute difference between // two numbers interpreted as signed/unsigned. // i.e trunc(abs(sext(Op0) - sext(Op1))) becomes abds(Op0, Op1) @@ -864,6 +898,13 @@ enum NodeType { STRICT_FP16_TO_FP, STRICT_FP_TO_FP16, + /// BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions + /// and truncation for bfloat16. These nodes form a semi-softened interface + /// for dealing with bf16 (as an i16), which is often a storage-only type but + /// has native conversions. + BF16_TO_FP, + FP_TO_BF16, + /// Perform various unary floating-point operations inspired by libm. For /// FPOWI, the result is undefined if if the integer operand doesn't fit into /// sizeof(int). @@ -1324,18 +1365,18 @@ static const int LAST_INDEXED_MODE = POST_DEC + 1; /// MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's /// index parameter when calculating addresses. /// -/// SIGNED_SCALED Addr = Base + ((signed)Index * sizeof(element)) -/// SIGNED_UNSCALED Addr = Base + (signed)Index -/// UNSIGNED_SCALED Addr = Base + ((unsigned)Index * sizeof(element)) -/// UNSIGNED_UNSCALED Addr = Base + (unsigned)Index -enum MemIndexType { - SIGNED_SCALED = 0, - SIGNED_UNSCALED, - UNSIGNED_SCALED, - UNSIGNED_UNSCALED -}; +/// SIGNED_SCALED Addr = Base + ((signed)Index * Scale) +/// UNSIGNED_SCALED Addr = Base + ((unsigned)Index * Scale) +/// +/// NOTE: The value of Scale is typically only known to the node owning the +/// IndexType, with a value of 1 the equivalent of being unscaled. +enum MemIndexType { SIGNED_SCALED = 0, UNSIGNED_SCALED }; -static const int LAST_MEM_INDEX_TYPE = UNSIGNED_UNSCALED + 1; +static const int LAST_MEM_INDEX_TYPE = UNSIGNED_SCALED + 1; + +inline bool isIndexTypeSigned(MemIndexType IndexType) { + return IndexType == SIGNED_SCALED; +} //===--------------------------------------------------------------------===// /// LoadExtType enum - This enum defines the three variants of LOADEXT diff --git a/llvm/include/llvm/CodeGen/IntrinsicLowering.h b/llvm/include/llvm/CodeGen/IntrinsicLowering.h index 06512f2dc560..0b327a34ca09 100644 --- a/llvm/include/llvm/CodeGen/IntrinsicLowering.h +++ b/llvm/include/llvm/CodeGen/IntrinsicLowering.h @@ -15,8 +15,6 @@ #ifndef LLVM_CODEGEN_INTRINSICLOWERING_H #define LLVM_CODEGEN_INTRINSICLOWERING_H -#include "llvm/IR/Intrinsics.h" - namespace llvm { class CallInst; class DataLayout; diff --git a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h index c692dbc2199e..e5794966ce63 100644 --- a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h +++ b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h @@ -17,8 +17,8 @@ #define LLVM_CODEGEN_LAZYMACHINEBLOCKFREQUENCYINFO_H #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" -#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h index 51ffe2807434..92e35c9a4ab9 100644 --- a/llvm/include/llvm/CodeGen/LiveInterval.h +++ b/llvm/include/llvm/CodeGen/LiveInterval.h @@ -227,6 +227,14 @@ namespace llvm { const_vni_iterator vni_begin() const { return valnos.begin(); } const_vni_iterator vni_end() const { return valnos.end(); } + iterator_range<vni_iterator> vnis() { + return make_range(vni_begin(), vni_end()); + } + + iterator_range<const_vni_iterator> vnis() const { + return make_range(vni_begin(), vni_end()); + } + /// Constructs a new LiveRange object. LiveRange(bool UseSegmentSet = false) : segmentSet(UseSegmentSet ? std::make_unique<SegmentSet>() @@ -625,10 +633,8 @@ namespace llvm { // if the Seg is lower find first segment that is above Idx using binary // search if (Seg->end <= *Idx) { - Seg = std::upper_bound( - ++Seg, EndSeg, *Idx, - [=](std::remove_reference_t<decltype(*Idx)> V, - const std::remove_reference_t<decltype(*Seg)> &S) { + Seg = + std::upper_bound(++Seg, EndSeg, *Idx, [=](auto V, const auto &S) { return V < S.end; }); if (Seg == EndSeg) diff --git a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h index 3b6a4a379d72..81003455da42 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h @@ -43,7 +43,7 @@ class LiveIntervalUnion { // A set of live virtual register segments that supports fast insertion, // intersection, and removal. // Mapping SlotIndex intervals to virtual register numbers. - using LiveSegments = IntervalMap<SlotIndex, LiveInterval*>; + using LiveSegments = IntervalMap<SlotIndex, const LiveInterval *>; public: // SegmentIter can advance to the next segment ordered by starting position @@ -88,10 +88,10 @@ public: bool changedSince(unsigned tag) const { return tag != Tag; } // Add a live virtual register to this union and merge its segments. - void unify(LiveInterval &VirtReg, const LiveRange &Range); + void unify(const LiveInterval &VirtReg, const LiveRange &Range); // Remove a live virtual register's segments from this union. - void extract(LiveInterval &VirtReg, const LiveRange &Range); + void extract(const LiveInterval &VirtReg, const LiveRange &Range); // Remove all inserted virtual registers. void clear() { Segments.clear(); ++Tag; } @@ -105,7 +105,7 @@ public: #endif // Get any virtual register that is assign to this physical unit - LiveInterval *getOneVReg() const; + const LiveInterval *getOneVReg() const; /// Query interferences between a single live virtual register and a live /// interval union. @@ -114,7 +114,7 @@ public: const LiveRange *LR = nullptr; LiveRange::const_iterator LRI; ///< current position in LR ConstSegmentIter LiveUnionI; ///< current position in LiveUnion - SmallVector<LiveInterval *, 4> InterferingVRegs; + SmallVector<const LiveInterval *, 4> InterferingVRegs; bool CheckedFirstInterference = false; bool SeenAllInterferences = false; unsigned Tag = 0; @@ -125,7 +125,7 @@ public: unsigned collectInterferingVRegs(unsigned MaxInterferingRegs); // Was this virtual register visited during collectInterferingVRegs? - bool isSeenInterference(LiveInterval *VirtReg) const; + bool isSeenInterference(const LiveInterval *VirtReg) const; public: Query() = default; @@ -159,7 +159,7 @@ public: bool checkInterference() { return collectInterferingVRegs(1); } // Vector generated by collectInterferingVRegs. - const SmallVectorImpl<LiveInterval *> &interferingVRegs( + const SmallVectorImpl<const LiveInterval *> &interferingVRegs( unsigned MaxInterferingRegs = std::numeric_limits<unsigned>::max()) { if (!SeenAllInterferences || MaxInterferingRegs < InterferingVRegs.size()) collectInterferingVRegs(MaxInterferingRegs); diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h index fa08166791b0..b832eaa37305 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervals.h +++ b/llvm/include/llvm/CodeGen/LiveIntervals.h @@ -374,7 +374,7 @@ class VirtRegMap; /// /// Returns false if \p LI doesn't cross any register mask instructions. In /// that case, the bit vector is not filled in. - bool checkRegMaskInterference(LiveInterval &LI, + bool checkRegMaskInterference(const LiveInterval &LI, BitVector &UsableRegs); // Register unit functions. diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h index 99ba1a28c934..27285d63aa83 100644 --- a/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -32,6 +32,7 @@ #include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/MC/MCRegister.h" #include "llvm/MC/MCRegisterInfo.h" #include <cassert> #include <utility> @@ -39,6 +40,7 @@ namespace llvm { class MachineInstr; +class MachineFunction; class MachineOperand; class MachineRegisterInfo; class raw_ostream; diff --git a/llvm/include/llvm/CodeGen/LiveRangeCalc.h b/llvm/include/llvm/CodeGen/LiveRangeCalc.h index 31efd6e37e01..895ecff18f89 100644 --- a/llvm/include/llvm/CodeGen/LiveRangeCalc.h +++ b/llvm/include/llvm/CodeGen/LiveRangeCalc.h @@ -31,7 +31,6 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/SlotIndexes.h" -#include "llvm/MC/LaneBitmask.h" #include <utility> namespace llvm { diff --git a/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/llvm/include/llvm/CodeGen/LiveRangeEdit.h index d80522f5bdac..c6efa7b30d71 100644 --- a/llvm/include/llvm/CodeGen/LiveRangeEdit.h +++ b/llvm/include/llvm/CodeGen/LiveRangeEdit.h @@ -66,7 +66,7 @@ public: }; private: - LiveInterval *Parent; + const LiveInterval *const Parent; SmallVectorImpl<Register> &NewRegs; MachineRegisterInfo &MRI; LiveIntervals &LIS; @@ -129,7 +129,7 @@ public: /// be done. This could be the case if called before Regalloc. /// @param deadRemats The collection of all the instructions defining an /// original reg and are dead after remat. - LiveRangeEdit(LiveInterval *parent, SmallVectorImpl<Register> &newRegs, + LiveRangeEdit(const LiveInterval *parent, SmallVectorImpl<Register> &newRegs, MachineFunction &MF, LiveIntervals &lis, VirtRegMap *vrm, Delegate *delegate = nullptr, SmallPtrSet<MachineInstr *, 32> *deadRemats = nullptr) @@ -141,7 +141,7 @@ public: ~LiveRangeEdit() override { MRI.resetDelegate(this); } - LiveInterval &getParent() const { + const LiveInterval &getParent() const { assert(Parent && "No parent LiveInterval"); return *Parent; } @@ -193,11 +193,11 @@ public: /// Remat - Information needed to rematerialize at a specific location. struct Remat { - VNInfo *ParentVNI; // parent_'s value at the remat location. + const VNInfo *const ParentVNI; // parent_'s value at the remat location. MachineInstr *OrigMI = nullptr; // Instruction defining OrigVNI. It contains // the real expr for remat. - explicit Remat(VNInfo *ParentVNI) : ParentVNI(ParentVNI) {} + explicit Remat(const VNInfo *ParentVNI) : ParentVNI(ParentVNI) {} }; /// allUsesAvailableAt - Return true if all registers used by OrigMI at diff --git a/llvm/include/llvm/CodeGen/LiveRegMatrix.h b/llvm/include/llvm/CodeGen/LiveRegMatrix.h index fc67bce329ab..9e28e4d243c2 100644 --- a/llvm/include/llvm/CodeGen/LiveRegMatrix.h +++ b/llvm/include/llvm/CodeGen/LiveRegMatrix.h @@ -104,7 +104,8 @@ public: /// If this function returns IK_Free, it is legal to assign(VirtReg, PhysReg). /// When there is more than one kind of interference, the InterferenceKind /// with the highest enum value is returned. - InterferenceKind checkInterference(LiveInterval &VirtReg, MCRegister PhysReg); + InterferenceKind checkInterference(const LiveInterval &VirtReg, + MCRegister PhysReg); /// Check for interference in the segment [Start, End) that may prevent /// assignment to PhysReg. If this function returns true, there is @@ -116,12 +117,12 @@ public: /// Assign VirtReg to PhysReg. /// This will mark VirtReg's live range as occupied in the LiveRegMatrix and /// update VirtRegMap. The live range is expected to be available in PhysReg. - void assign(LiveInterval &VirtReg, MCRegister PhysReg); + void assign(const LiveInterval &VirtReg, MCRegister PhysReg); /// Unassign VirtReg from its PhysReg. /// Assuming that VirtReg was previously assigned to a PhysReg, this undoes /// the assignment and updates VirtRegMap accordingly. - void unassign(LiveInterval &VirtReg); + void unassign(const LiveInterval &VirtReg); /// Returns true if the given \p PhysReg has any live intervals assigned. bool isPhysRegUsed(MCRegister PhysReg) const; @@ -136,13 +137,14 @@ public: /// Check for regmask interference only. /// Return true if VirtReg crosses a regmask operand that clobbers PhysReg. /// If PhysReg is null, check if VirtReg crosses any regmask operands. - bool checkRegMaskInterference(LiveInterval &VirtReg, + bool checkRegMaskInterference(const LiveInterval &VirtReg, MCRegister PhysReg = MCRegister::NoRegister); /// Check for regunit interference only. /// Return true if VirtReg overlaps a fixed assignment of one of PhysRegs's /// register units. - bool checkRegUnitInterference(LiveInterval &VirtReg, MCRegister PhysReg); + bool checkRegUnitInterference(const LiveInterval &VirtReg, + MCRegister PhysReg); /// Query a line of the assigned virtual register matrix directly. /// Use MCRegUnitIterator to enumerate all regunits in the desired PhysReg. diff --git a/llvm/include/llvm/CodeGen/LiveStacks.h b/llvm/include/llvm/CodeGen/LiveStacks.h index 1cbdb8bd86bd..26f30fb4d088 100644 --- a/llvm/include/llvm/CodeGen/LiveStacks.h +++ b/llvm/include/llvm/CodeGen/LiveStacks.h @@ -18,13 +18,17 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/InitializePasses.h" -#include "llvm/Pass.h" +#include "llvm/PassRegistry.h" #include <cassert> #include <map> #include <unordered_map> namespace llvm { +class AnalysisUsage; +class MachineFunction; +class Module; +class raw_ostream; class TargetRegisterClass; class TargetRegisterInfo; diff --git a/llvm/include/llvm/CodeGen/LiveVariables.h b/llvm/include/llvm/CodeGen/LiveVariables.h index dee316677b25..aa198527415d 100644 --- a/llvm/include/llvm/CodeGen/LiveVariables.h +++ b/llvm/include/llvm/CodeGen/LiveVariables.h @@ -37,6 +37,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/InitializePasses.h" +#include "llvm/PassRegistry.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/MIRFSDiscriminator.h b/llvm/include/llvm/CodeGen/MIRFSDiscriminator.h index deb6b37a9bcf..3bbcfd63e3aa 100644 --- a/llvm/include/llvm/CodeGen/MIRFSDiscriminator.h +++ b/llvm/include/llvm/CodeGen/MIRFSDiscriminator.h @@ -17,29 +17,16 @@ #ifndef LLVM_CODEGEN_MIRFSDISCRIMINATOR_H #define LLVM_CODEGEN_MIRFSDISCRIMINATOR_H -#include "llvm/Analysis/ProfileSummaryInfo.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" -#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" -#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" -#include "llvm/CodeGen/MachinePostDominators.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" -#include "llvm/InitializePasses.h" -#include "llvm/ProfileData/InstrProf.h" -#include "llvm/ProfileData/SampleProf.h" -#include "llvm/ProfileData/SampleProfReader.h" +#include "llvm/Support/Discriminator.h" #include <cassert> +#include <cstdint> namespace llvm { +class MachineFunction; using namespace sampleprof; class MIRAddFSDiscriminators : public MachineFunctionPass { diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h index a7c69e2d43ef..aa9891a80a32 100644 --- a/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h +++ b/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h @@ -17,13 +17,20 @@ #ifndef LLVM_CODEGEN_MIRPARSER_MIRPARSER_H #define LLVM_CODEGEN_MIRPARSER_MIRPARSER_H -#include "llvm/IR/Module.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLForwardCompat.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include <functional> #include <memory> namespace llvm { class Function; +class LLVMContext; +class MemoryBuffer; +class Module; class MIRParserImpl; class MachineModuleInfo; class SMDiagnostic; diff --git a/llvm/include/llvm/CodeGen/MIRSampleProfile.h b/llvm/include/llvm/CodeGen/MIRSampleProfile.h index 2503524ccfdf..f54c4b5891be 100644 --- a/llvm/include/llvm/CodeGen/MIRSampleProfile.h +++ b/llvm/include/llvm/CodeGen/MIRSampleProfile.h @@ -14,29 +14,17 @@ #ifndef LLVM_CODEGEN_MIRSAMPLEPROFILE_H #define LLVM_CODEGEN_MIRSAMPLEPROFILE_H -#include "llvm/Analysis/ProfileSummaryInfo.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" -#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" -#include "llvm/CodeGen/MachinePostDominators.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" -#include "llvm/InitializePasses.h" -#include "llvm/ProfileData/InstrProf.h" -#include "llvm/ProfileData/SampleProf.h" -#include "llvm/ProfileData/SampleProfReader.h" - -#include <cassert> +#include "llvm/Support/Discriminator.h" +#include <memory> +#include <string> namespace llvm { +class AnalysisUsage; +class MachineBlockFrequencyInfo; +class MachineFunction; +class Module; using namespace sampleprof; diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h index 02eb5d24271d..25247437b641 100644 --- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -605,7 +605,7 @@ struct MachineFrameInfo { bool AdjustsStack = false; bool HasCalls = false; StringValue StackProtector; - // TODO: Serialize FunctionContextIdx + StringValue FunctionContext; unsigned MaxCallFrameSize = ~0u; ///< ~0u means: not computed yet. unsigned CVBytesOfCalleeSavedRegisters = 0; bool HasOpaqueSPAdjustment = false; @@ -626,6 +626,7 @@ struct MachineFrameInfo { MaxAlignment == Other.MaxAlignment && AdjustsStack == Other.AdjustsStack && HasCalls == Other.HasCalls && StackProtector == Other.StackProtector && + FunctionContext == Other.FunctionContext && MaxCallFrameSize == Other.MaxCallFrameSize && CVBytesOfCalleeSavedRegisters == Other.CVBytesOfCalleeSavedRegisters && @@ -651,6 +652,8 @@ template <> struct MappingTraits<MachineFrameInfo> { YamlIO.mapOptional("hasCalls", MFI.HasCalls, false); YamlIO.mapOptional("stackProtector", MFI.StackProtector, StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("functionContext", MFI.FunctionContext, + StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize, (unsigned)~0); YamlIO.mapOptional("cvBytesOfCalleeSavedRegisters", MFI.CVBytesOfCalleeSavedRegisters, 0U); @@ -694,6 +697,13 @@ struct MachineFunction { // Register information bool TracksRegLiveness = false; bool HasWinCFI = false; + + bool CallsEHReturn = false; + bool CallsUnwindInit = false; + bool HasEHCatchret = false; + bool HasEHScopes = false; + bool HasEHFunclets = false; + bool FailsVerification = false; bool TracksDebugUserValues = false; std::vector<VirtualRegisterDefinition> VirtualRegisters; @@ -724,6 +734,13 @@ template <> struct MappingTraits<MachineFunction> { YamlIO.mapOptional("failedISel", MF.FailedISel, false); YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness, false); YamlIO.mapOptional("hasWinCFI", MF.HasWinCFI, false); + + YamlIO.mapOptional("callsEHReturn", MF.CallsEHReturn, false); + YamlIO.mapOptional("callsUnwindInit", MF.CallsUnwindInit, false); + YamlIO.mapOptional("hasEHCatchret", MF.HasEHCatchret, false); + YamlIO.mapOptional("hasEHScopes", MF.HasEHScopes, false); + YamlIO.mapOptional("hasEHFunclets", MF.HasEHFunclets, false); + YamlIO.mapOptional("failsVerification", MF.FailsVerification, false); YamlIO.mapOptional("tracksDebugUserValues", MF.TracksDebugUserValues, false); diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index 638b6732a543..ddfbd4018590 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -14,9 +14,9 @@ #define LLVM_CODEGEN_MACHINEBASICBLOCK_H #include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/SparseBitVector.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/ADT/SparseBitVector.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBundleIterator.h" #include "llvm/IR/DebugLoc.h" @@ -24,7 +24,6 @@ #include "llvm/Support/BranchProbability.h" #include <cassert> #include <cstdint> -#include <functional> #include <iterator> #include <string> #include <vector> @@ -110,10 +109,10 @@ public: private: using Instructions = ilist<MachineInstr, ilist_sentinel_tracking<true>>; - Instructions Insts; const BasicBlock *BB; int Number; MachineFunction *xParent; + Instructions Insts; /// Keep track of the predecessor / successor basic blocks. std::vector<MachineBasicBlock *> Predecessors; @@ -205,6 +204,12 @@ public: /// to an LLVM basic block. const BasicBlock *getBasicBlock() const { return BB; } + /// Remove the reference to the underlying IR BasicBlock. This is for + /// reduction tools and should generally not be used. + void clearBasicBlock() { + BB = nullptr; + } + /// Return the name of the corresponding LLVM basic block, or an empty string. StringRef getName() const; @@ -241,6 +246,7 @@ public: MachineInstrBundleIterator<const MachineInstr, true>; unsigned size() const { return (unsigned)Insts.size(); } + bool sizeWithoutDebugLargerThan(unsigned Limit) const; bool empty() const { return Insts.empty(); } MachineInstr &instr_front() { return Insts.front(); } @@ -400,7 +406,7 @@ public: // Iteration support for live in sets. These sets are kept in sorted // order by their register number. using livein_iterator = LiveInVector::const_iterator; -#ifndef NDEBUG + /// Unlike livein_begin, this method does not check that the liveness /// information is accurate. Still for debug purposes it may be useful /// to have iterators that won't assert if the liveness information @@ -409,7 +415,7 @@ public: iterator_range<livein_iterator> liveins_dbg() const { return make_range(livein_begin_dbg(), livein_end()); } -#endif + livein_iterator livein_begin() const; livein_iterator livein_end() const { return LiveIns.end(); } bool livein_empty() const { return LiveIns.empty(); } @@ -731,6 +737,15 @@ public: /// other block. bool isLayoutSuccessor(const MachineBasicBlock *MBB) const; + /// Return the successor of this block if it has a single successor. + /// Otherwise return a null pointer. + /// + const MachineBasicBlock *getSingleSuccessor() const; + MachineBasicBlock *getSingleSuccessor() { + return const_cast<MachineBasicBlock *>( + static_cast<const MachineBasicBlock *>(this)->getSingleSuccessor()); + } + /// Return the fallthrough block if the block can implicitly /// transfer control to the block after it by falling off the end of /// it. This should return null if it can reach the block after @@ -1087,6 +1102,11 @@ public: IrrLoopHeaderWeight = Weight; } + /// Return probability of the edge from this block to MBB. This method should + /// NOT be called directly, but by using getEdgeProbability method from + /// MachineBranchProbabilityInfo class. + BranchProbability getSuccProbability(const_succ_iterator Succ) const; + private: /// Return probability iterator corresponding to the I successor iterator. probability_iterator getProbabilityIterator(succ_iterator I); @@ -1096,11 +1116,6 @@ private: friend class MachineBranchProbabilityInfo; friend class MIPrinter; - /// Return probability of the edge from this block to MBB. This method should - /// NOT be called directly, but by using getEdgeProbability method from - /// MachineBranchProbabilityInfo class. - BranchProbability getSuccProbability(const_succ_iterator Succ) const; - // Methods used to maintain doubly linked list of blocks... friend struct ilist_callback_traits<MachineBasicBlock>; diff --git a/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h index 7e7e0a9c477a..bd544421bc0f 100644 --- a/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h +++ b/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h @@ -16,8 +16,6 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/Pass.h" #include "llvm/Support/BranchProbability.h" -#include <climits> -#include <numeric> namespace llvm { diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h index 67544779f34c..68c95679d466 100644 --- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h @@ -34,6 +34,10 @@ enum class MachineCombinerPattern { REASSOC_XY_BCA, REASSOC_XY_BAC, + // These are patterns used to reduce the length of dependence chain. + SUBADD_OP1, + SUBADD_OP2, + // These are multiply-add patterns matched by the AArch64 machine combiner. MULADDW_OP1, MULADDW_OP2, diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h index d3816bbc0780..3f89f2076d50 100644 --- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h +++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h @@ -15,8 +15,9 @@ #define LLVM_CODEGEN_MACHINECYCLEANALYSIS_H #include "llvm/ADT/GenericCycleInfo.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineSSAContext.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" namespace llvm { @@ -26,6 +27,29 @@ extern template class GenericCycle<MachineSSAContext>; using MachineCycleInfo = GenericCycleInfo<MachineSSAContext>; using MachineCycle = MachineCycleInfo::CycleT; +/// Legacy analysis pass which computes a \ref MachineCycleInfo. +class MachineCycleInfoWrapperPass : public MachineFunctionPass { + MachineFunction *F = nullptr; + MachineCycleInfo CI; + +public: + static char ID; + + MachineCycleInfoWrapperPass(); + + MachineCycleInfo &getCycleInfo() { return CI; } + const MachineCycleInfo &getCycleInfo() const { return CI; } + + bool runOnMachineFunction(MachineFunction &F) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; + void print(raw_ostream &OS, const Module *M = nullptr) const override; +}; + +// TODO: add this function to GenericCycle template after implementing IR +// version. +bool isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I); + } // end namespace llvm #endif // LLVM_CODEGEN_MACHINECYCLEANALYSIS_H diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h index f749e9ff7e0a..30c18ef410fa 100644 --- a/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/llvm/include/llvm/CodeGen/MachineDominators.h @@ -19,12 +19,17 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBundleIterator.h" #include "llvm/Support/GenericDomTree.h" #include "llvm/Support/GenericDomTreeConstruction.h" #include <cassert> #include <memory> namespace llvm { +class AnalysisUsage; +class MachineFunction; +class Module; +class raw_ostream; template <> inline void DominatorTreeBase<MachineBasicBlock, false>::addRoot( diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h index 864ca73180af..7ea731b46655 100644 --- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -16,7 +16,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/Register.h" #include "llvm/Support/Alignment.h" -#include "llvm/Support/DataTypes.h" #include <cassert> #include <vector> @@ -335,10 +334,13 @@ private: /// Not null, if shrink-wrapping found a better place for the epilogue. MachineBasicBlock *Restore = nullptr; + /// Size of the UnsafeStack Frame + uint64_t UnsafeStackSize = 0; + public: - explicit MachineFrameInfo(unsigned StackAlignment, bool StackRealignable, + explicit MachineFrameInfo(Align StackAlignment, bool StackRealignable, bool ForcedRealign) - : StackAlignment(assumeAligned(StackAlignment)), + : StackAlignment(StackAlignment), StackRealignable(StackRealignable), ForcedRealign(ForcedRealign) {} MachineFrameInfo(const MachineFrameInfo &) = delete; @@ -360,6 +362,7 @@ public: /// This object is used for SjLj exceptions. int getFunctionContextIndex() const { return FunctionContextIdx; } void setFunctionContextIndex(int I) { FunctionContextIdx = I; } + bool hasFunctionContextIndex() const { return FunctionContextIdx != -1; } /// This method may be called any time after instruction /// selection is complete to determine if there is a call to @@ -385,6 +388,20 @@ public: bool hasPatchPoint() const { return HasPatchPoint; } void setHasPatchPoint(bool s = true) { HasPatchPoint = s; } + /// Return true if this function requires a split stack prolog, even if it + /// uses no stack space. This is only meaningful for functions where + /// MachineFunction::shouldSplitStack() returns true. + // + // For non-leaf functions we have to allow for the possibility that the call + // is to a non-split function, as in PR37807. This function could also take + // the address of a non-split function. When the linker tries to adjust its + // non-existent prologue, it would fail with an error. Mark the object file so + // that such failures are not errors. See this Go language bug-report + // https://go-review.googlesource.com/c/go/+/148819/ + bool needsSplitStackProlog() const { + return getStackSize() != 0 || hasTailCall(); + } + /// Return the minimum frame object index. int getObjectIndexBegin() const { return -NumFixedObjects; } @@ -488,6 +505,14 @@ public: return Objects[ObjectIdx+NumFixedObjects].Alloca; } + /// Remove the underlying Alloca of the specified stack object if it + /// exists. This generally should not be used and is for reduction tooling. + void clearObjectAllocation(int ObjectIdx) { + assert(unsigned(ObjectIdx + NumFixedObjects) < Objects.size() && + "Invalid Object Idx!"); + Objects[ObjectIdx + NumFixedObjects].Alloca = nullptr; + } + /// Return the assigned stack offset of the specified object /// from the incoming stack pointer. int64_t getObjectOffset(int ObjectIdx) const { @@ -773,6 +798,9 @@ public: MachineBasicBlock *getRestorePoint() const { return Restore; } void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; } + uint64_t getUnsafeStackSize() const { return UnsafeStackSize; } + void setUnsafeStackSize(uint64_t Size) { UnsafeStackSize = Size; } + /// Return a set of physical registers that are pristine. /// /// Pristine registers hold a value that is useless to the current function, diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index c4767a51b094..fc1188186ac4 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -103,6 +103,22 @@ struct MachineFunctionInfo { static Ty *create(BumpPtrAllocator &Allocator, MachineFunction &MF) { return new (Allocator.Allocate<Ty>()) Ty(MF); } + + template <typename Ty> + static Ty *create(BumpPtrAllocator &Allocator, const Ty &MFI) { + return new (Allocator.Allocate<Ty>()) Ty(MFI); + } + + /// Make a functionally equivalent copy of this MachineFunctionInfo in \p MF. + /// This requires remapping MachineBasicBlock references from the original + /// parent to values in the new function. Targets may assume that virtual + /// register and frame index values are preserved in the new function. + virtual MachineFunctionInfo * + clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, + const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) + const { + return nullptr; + } }; /// Properties which a MachineFunction may have at a given point in time. @@ -277,12 +293,6 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction { // numbered and this vector keeps track of the mapping from ID's to MBB's. std::vector<MachineBasicBlock*> MBBNumbering; - // Unary encoding of basic block symbols is used to reduce size of ".strtab". - // Basic block number 'i' gets a prefix of length 'i'. The ith character also - // denotes the type of basic block number 'i'. Return blocks are marked with - // 'r', landing pads with 'l' and regular blocks with 'a'. - std::vector<char> BBSectionsSymbolPrefix; - // Pool-allocate MachineFunction-lifetime and IR objects. BumpPtrAllocator Allocator; @@ -537,8 +547,13 @@ public: /// the copied value; or for parameters, creates a DBG_PHI on entry. /// May insert instructions into the entry block! /// \p MI The copy-like instruction to salvage. + /// \p DbgPHICache A container to cache already-solved COPYs. /// \returns An instruction/operand pair identifying the defining value. - DebugInstrOperandPair salvageCopySSA(MachineInstr &MI); + DebugInstrOperandPair + salvageCopySSA(MachineInstr &MI, + DenseMap<Register, DebugInstrOperandPair> &DbgPHICache); + + DebugInstrOperandPair salvageCopySSAImpl(MachineInstr &MI); /// Finalise any partially emitted debug instructions. These are DBG_INSTR_REF /// instructions where we only knew the vreg of the value they use, not the @@ -747,6 +762,21 @@ public: return const_cast<MachineFunction*>(this)->getInfo<Ty>(); } + template <typename Ty> Ty *cloneInfo(const Ty &Old) { + assert(!MFInfo); + MFInfo = Ty::template create<Ty>(Allocator, Old); + return static_cast<Ty *>(MFInfo); + } + + MachineFunctionInfo *cloneInfoFrom( + const MachineFunction &OrigMF, + const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) { + assert(!MFInfo && "new function already has MachineFunctionInfo"); + if (!OrigMF.MFInfo) + return nullptr; + return OrigMF.MFInfo->clone(Allocator, *this, Src2DstMBB); + } + /// Returns the denormal handling type for the default rounding mode of the /// function. DenormalMode getDenormalMode(const fltSemantics &FPType) const; @@ -1101,12 +1131,6 @@ public: /// Add a cleanup action for a landing pad. void addCleanup(MachineBasicBlock *LandingPad); - void addSEHCatchHandler(MachineBasicBlock *LandingPad, const Function *Filter, - const BlockAddress *RecoverBA); - - void addSEHCleanupHandler(MachineBasicBlock *LandingPad, - const Function *Cleanup); - /// Return the type id for the specified typeinfo. This is function wide. unsigned getTypeIDFor(const GlobalValue *TI); @@ -1116,6 +1140,11 @@ public: /// Map the landing pad's EH symbol to the call site indexes. void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef<unsigned> Sites); + /// Return if there is any wasm exception handling. + bool hasAnyWasmLandingPadIndex() const { + return !WasmLPadToIndexMap.empty(); + } + /// Map the landing pad to its index. Used for Wasm exception handling. void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index) { WasmLPadToIndexMap[LPad] = Index; @@ -1132,6 +1161,10 @@ public: return WasmLPadToIndexMap.lookup(LPad); } + bool hasAnyCallSiteLandingPad() const { + return !LPadToCallSiteMap.empty(); + } + /// Get the call site indexes for a landing pad EH symbol. SmallVectorImpl<unsigned> &getCallSiteLandingPad(MCSymbol *Sym) { assert(hasCallSiteLandingPad(Sym) && @@ -1144,6 +1177,10 @@ public: return !LPadToCallSiteMap[Sym].empty(); } + bool hasAnyCallSiteLabel() const { + return !CallSiteMap.empty(); + } + /// Map the begin label for a call site. void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site) { CallSiteMap[BeginLabel] = Site; @@ -1220,10 +1257,6 @@ public: void copyCallSiteInfo(const MachineInstr *Old, const MachineInstr *New); - const std::vector<char> &getBBSectionsSymbolPrefix() const { - return BBSectionsSymbolPrefix; - } - /// Move the call site info from \p Old to \New call site info. This function /// is used when we are replacing one call instruction with another one to /// the same callee. diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index 2893e138a95c..acc4c9a24c01 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -26,7 +26,6 @@ #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" -#include "llvm/IR/PseudoProbe.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/ArrayRecycler.h" @@ -38,6 +37,9 @@ namespace llvm { +class DILabel; +class Instruction; +class MDNode; class AAResults; template <typename T> class ArrayRef; class DIExpression; @@ -96,7 +98,7 @@ public: FmContract = 1 << 8, // Instruction supports Fast math // contraction operations like fma. FmAfn = 1 << 9, // Instruction may map to Fast math - // instrinsic approximation. + // intrinsic approximation. FmReassoc = 1 << 10, // Instruction supports Fast math // reassociation of operand order. NoUWrap = 1 << 11, // Instruction supports binary operator @@ -586,8 +588,7 @@ public: /// Return true if operand \p OpIdx is a subregister index. bool isOperandSubregIdx(unsigned OpIdx) const { - assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && - "Expected MO_Immediate operand type."); + assert(getOperand(OpIdx).isImm() && "Expected MO_Immediate operand type."); if (isExtractSubreg() && OpIdx == 2) return true; if (isInsertSubreg() && OpIdx == 3) @@ -810,6 +811,12 @@ public: return hasProperty(MCID::Pseudo, Type); } + /// Return true if this instruction doesn't produce any output in the form of + /// executable instructions. + bool isMetaInstruction(QueryType Type = IgnoreBundle) const { + return hasProperty(MCID::Meta, Type); + } + bool isReturn(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Return, Type); } @@ -1306,30 +1313,6 @@ public: getOperand(0).getSubReg() == getOperand(1).getSubReg(); } - /// Return true if this instruction doesn't produce any output in the form of - /// executable instructions. - bool isMetaInstruction() const { - switch (getOpcode()) { - default: - return false; - case TargetOpcode::IMPLICIT_DEF: - case TargetOpcode::KILL: - case TargetOpcode::CFI_INSTRUCTION: - case TargetOpcode::EH_LABEL: - case TargetOpcode::GC_LABEL: - case TargetOpcode::DBG_VALUE: - case TargetOpcode::DBG_VALUE_LIST: - case TargetOpcode::DBG_INSTR_REF: - case TargetOpcode::DBG_PHI: - case TargetOpcode::DBG_LABEL: - case TargetOpcode::LIFETIME_START: - case TargetOpcode::LIFETIME_END: - case TargetOpcode::PSEUDO_PROBE: - case TargetOpcode::ARITH_FENCE: - return true; - } - } - /// Return true if this is a transient instruction that is either very likely /// to be eliminated during register allocation (such as copy-like /// instructions), or if this instruction doesn't have an execution-time cost. @@ -1744,7 +1727,7 @@ public: /// Erase an operand from an instruction, leaving it with one /// fewer operand than it started with. - void RemoveOperand(unsigned OpNo); + void removeOperand(unsigned OpNo); /// Clear this MachineInstr's memory reference descriptor list. This resets /// the memrefs to their most conservative state. This should be used only @@ -1863,12 +1846,12 @@ private: /// Unlink all of the register operands in this instruction from their /// respective use lists. This requires that the operands already be on their /// use lists. - void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); + void removeRegOperandsFromUseLists(MachineRegisterInfo&); /// Add all of the register operands in this instruction from their /// respective use lists. This requires that the operands not be on their /// use lists yet. - void AddRegOperandsToUseLists(MachineRegisterInfo&); + void addRegOperandsToUseLists(MachineRegisterInfo&); /// Slow path for hasProperty when we're dealing with a bundle. bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; diff --git a/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/llvm/include/llvm/CodeGen/MachineLoopInfo.h index c90f07096d02..daf0f18a7518 100644 --- a/llvm/include/llvm/CodeGen/MachineLoopInfo.h +++ b/llvm/include/llvm/CodeGen/MachineLoopInfo.h @@ -33,7 +33,6 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/IR/DebugLoc.h" -#include "llvm/Pass.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h index 00080b171974..41574d8d556a 100644 --- a/llvm/include/llvm/CodeGen/MachineMemOperand.h +++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -31,14 +31,13 @@ class MDNode; class raw_ostream; class MachineFunction; class ModuleSlotTracker; +class TargetInstrInfo; /// This class contains a discriminated union of information about pointers in /// memory operands, relating them back to LLVM IR or to virtual locations (such /// as frame indices) that are exposed during codegen. struct MachinePointerInfo { /// This is the IR pointer value for the access, or it is null if unknown. - /// If this is null, then the access is to a pointer in the default address - /// space. PointerUnion<const Value *, const PseudoSourceValue *> V; /// Offset - This is an offset from the base Value*. diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/llvm/include/llvm/CodeGen/MachineModuleInfo.h index c07606e89374..cdd0073749d3 100644 --- a/llvm/include/llvm/CodeGen/MachineModuleInfo.h +++ b/llvm/include/llvm/CodeGen/MachineModuleInfo.h @@ -30,12 +30,10 @@ #ifndef LLVM_CODEGEN_MACHINEMODULEINFO_H #define LLVM_CODEGEN_MACHINEMODULEINFO_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/IR/PassManager.h" #include "llvm/MC/MCContext.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/Pass.h" #include <memory> #include <utility> @@ -46,9 +44,9 @@ namespace llvm { class BasicBlock; class Function; class LLVMTargetMachine; -class MMIAddrLabelMap; class MachineFunction; class Module; +class MCSymbol; //===----------------------------------------------------------------------===// /// This class can be derived from and used by targets to hold private @@ -106,10 +104,6 @@ class MachineModuleInfo { /// \} - /// This map keeps track of which symbol is being used for the specified - /// basic block's address of label. - MMIAddrLabelMap *AddrLabelSymbols; - // TODO: Ideally, what we'd like is to have a switch that allows emitting // synchronous (precise at call-sites only) CFA into .eh_frame. However, // even under this switch, we'd like .debug_frame to be precise when using @@ -123,22 +117,6 @@ class MachineModuleInfo { /// point. This is used to emit an undefined reference to _fltused. bool UsesMSVCFloatingPoint; - /// True if the module calls the __morestack function indirectly, as is - /// required under the large code model on x86. This is used to emit - /// a definition of a symbol, __morestack_addr, containing the address. See - /// comments in lib/Target/X86/X86FrameLowering.cpp for more details. - bool UsesMorestackAddr; - - /// True if the module contains split-stack functions. This is used to - /// emit .note.GNU-split-stack section as required by the linker for - /// special handling split-stack function calling no-split-stack function. - bool HasSplitStack; - - /// True if the module contains no-split-stack functions. This is used to - /// emit .note.GNU-no-split-stack section when it also contains split-stack - /// functions. - bool HasNosplitStack; - /// Maps IR Functions to their corresponding MachineFunctions. DenseMap<const Function*, std::unique_ptr<MachineFunction>> MachineFunctions; /// Next unique number available for a MachineFunction. @@ -184,6 +162,9 @@ public: /// Machine Function map. void deleteMachineFunctionFor(Function &F); + /// Add an externally created MachineFunction \p MF for \p F. + void insertFunction(const Function &F, std::unique_ptr<MachineFunction> &&MF); + /// Keep track of various per-module pieces of information for backends /// that would like to do so. template<typename Ty> @@ -200,55 +181,11 @@ public: /// Returns true if valid debug info is present. bool hasDebugInfo() const { return DbgInfoAvailable; } - void setDebugInfoAvailability(bool avail) { DbgInfoAvailable = avail; } bool usesMSVCFloatingPoint() const { return UsesMSVCFloatingPoint; } void setUsesMSVCFloatingPoint(bool b) { UsesMSVCFloatingPoint = b; } - bool usesMorestackAddr() const { - return UsesMorestackAddr; - } - - void setUsesMorestackAddr(bool b) { - UsesMorestackAddr = b; - } - - bool hasSplitStack() const { - return HasSplitStack; - } - - void setHasSplitStack(bool b) { - HasSplitStack = b; - } - - bool hasNosplitStack() const { - return HasNosplitStack; - } - - void setHasNosplitStack(bool b) { - HasNosplitStack = b; - } - - /// Return the symbol to be used for the specified basic block when its - /// address is taken. This cannot be its normal LBB label because the block - /// may be accessed outside its containing function. - MCSymbol *getAddrLabelSymbol(const BasicBlock *BB) { - return getAddrLabelSymbolToEmit(BB).front(); - } - - /// Return the symbol to be used for the specified basic block when its - /// address is taken. If other blocks were RAUW'd to this one, we may have - /// to emit them as well, return the whole set. - ArrayRef<MCSymbol *> getAddrLabelSymbolToEmit(const BasicBlock *BB); - - /// If the specified function has had any references to address-taken blocks - /// generated, but the block got deleted, return the symbol now so we can - /// emit it. This prevents emitting a reference to a symbol that has no - /// definition. - void takeDeletedSymbolsForFunction(const Function *F, - std::vector<MCSymbol*> &Result); - /// \name Exception Handling /// \{ diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h index eded28183ea2..c88e72cdc1d9 100644 --- a/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/llvm/include/llvm/CodeGen/MachineOperand.h @@ -13,15 +13,14 @@ #ifndef LLVM_CODEGEN_MACHINEOPERAND_H #define LLVM_CODEGEN_MACHINEOPERAND_H -#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/CodeGen/Register.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/LowLevelTypeImpl.h" #include <cassert> namespace llvm { +class LLT; class BlockAddress; class Constant; class ConstantFP; @@ -460,6 +459,16 @@ public: return !isUndef() && !isInternalRead() && (isUse() || getSubReg()); } + /// Return true if this operand can validly be appended to an arbitrary + /// operand list. i.e. this behaves like an implicit operand. + bool isValidExcessOperand() const { + if ((isReg() && isImplicit()) || isRegMask()) + return true; + + // Debug operands + return isMetadata() || isMCSymbol(); + } + //===--------------------------------------------------------------------===// // Mutators for Register Operands //===--------------------------------------------------------------------===// diff --git a/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h index 285b858c96cb..cb0998984dfb 100644 --- a/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h +++ b/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h @@ -15,8 +15,9 @@ #ifndef LLVM_CODEGEN_MACHINEOPTIMIZATIONREMARKEMITTER_H #define LLVM_CODEGEN_MACHINEOPTIMIZATIONREMARKEMITTER_H -#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Function.h" namespace llvm { class MachineBasicBlock; diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h index 08b76295dbf2..f968089e0de0 100644 --- a/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -15,11 +15,10 @@ #ifndef LLVM_CODEGEN_MACHINEOUTLINER_H #define LLVM_CODEGEN_MACHINEOUTLINER_H -#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" +#include <initializer_list> namespace llvm { namespace outliner { @@ -56,6 +55,55 @@ private: /// target. unsigned CallOverhead = 0; + /// Liveness information for this Candidate. Tracks from the end of the + /// block containing this Candidate to the beginning of its sequence. + /// + /// Optional. Can be used to fine-tune the cost model, or fine-tune legality + /// decisions. + LiveRegUnits FromEndOfBlockToStartOfSeq; + + /// Liveness information restricted to this Candidate's instruction sequence. + /// + /// Optional. Can be used to fine-tune the cost model, or fine-tune legality + /// decisions. + LiveRegUnits InSeq; + + /// True if FromEndOfBlockToStartOfSeq has been initialized. + bool FromEndOfBlockToStartOfSeqWasSet = false; + + /// True if InSeq has been initialized. + bool InSeqWasSet = false; + + /// Populate FromEndOfBlockToStartOfSeq with liveness information. + void initFromEndOfBlockToStartOfSeq(const TargetRegisterInfo &TRI) { + assert(MBB->getParent()->getRegInfo().tracksLiveness() && + "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (FromEndOfBlockToStartOfSeqWasSet) + return; + FromEndOfBlockToStartOfSeqWasSet = true; + FromEndOfBlockToStartOfSeq.init(TRI); + FromEndOfBlockToStartOfSeq.addLiveOuts(*MBB); + // Compute liveness from the end of the block up to the beginning of the + // outlining candidate. + for (auto &MI : make_range(MBB->rbegin(), + (MachineBasicBlock::reverse_iterator)front())) + FromEndOfBlockToStartOfSeq.stepBackward(MI); + } + + /// Populate InSeq with liveness information. + void initInSeq(const TargetRegisterInfo &TRI) { + assert(MBB->getParent()->getRegInfo().tracksLiveness() && + "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (InSeqWasSet) + return; + InSeqWasSet = true; + InSeq.init(TRI); + for (auto &MI : make_range(front(), std::next(back()))) + InSeq.accumulate(MI); + } + public: /// The index of this \p Candidate's \p OutlinedFunction in the list of /// \p OutlinedFunctions. @@ -65,26 +113,9 @@ public: /// from this point. Defined by the target. unsigned CallConstructionID = 0; - /// Contains physical register liveness information for the MBB containing - /// this \p Candidate. - /// - /// This is optionally used by the target to calculate more fine-grained - /// cost model information. - LiveRegUnits LRU; - - /// Contains the accumulated register liveness information for the - /// instructions in this \p Candidate. - /// - /// This is optionally used by the target to determine which registers have - /// been used across the sequence. - LiveRegUnits UsedInSequence; - /// Target-specific flags for this Candidate's MBB. unsigned Flags = 0x0; - /// True if initLRU has been called on this Candidate. - bool LRUWasSet = false; - /// Return the number of instructions in this Candidate. unsigned getLength() const { return Len; } @@ -109,6 +140,50 @@ public: MachineFunction *getMF() const { return MBB->getParent(); } MachineBasicBlock *getMBB() const { return MBB; } + /// \returns True if \p Reg is available from the end of the block to the + /// beginning of the sequence. + /// + /// This query considers the following range: + /// + /// in_seq_1 + /// in_seq_2 + /// ... + /// in_seq_n + /// not_in_seq_1 + /// ... + /// <end of block> + bool isAvailableAcrossAndOutOfSeq(Register Reg, + const TargetRegisterInfo &TRI) { + if (!FromEndOfBlockToStartOfSeqWasSet) + initFromEndOfBlockToStartOfSeq(TRI); + return FromEndOfBlockToStartOfSeq.available(Reg); + } + + /// \returns True if `isAvailableAcrossAndOutOfSeq` fails for any register + /// in \p Regs. + bool isAnyUnavailableAcrossOrOutOfSeq(std::initializer_list<Register> Regs, + const TargetRegisterInfo &TRI) { + if (!FromEndOfBlockToStartOfSeqWasSet) + initFromEndOfBlockToStartOfSeq(TRI); + return any_of(Regs, [&](Register Reg) { + return !FromEndOfBlockToStartOfSeq.available(Reg); + }); + } + + /// \returns True if \p Reg is available within the sequence itself. + /// + /// This query considers the following range: + /// + /// in_seq_1 + /// in_seq_2 + /// ... + /// in_seq_n + bool isAvailableInsideSeq(Register Reg, const TargetRegisterInfo &TRI) { + if (!InSeqWasSet) + initInSeq(TRI); + return InSeq.available(Reg); + } + /// The number of instructions that would be saved by outlining every /// candidate of this type. /// @@ -132,31 +207,6 @@ public: return getStartIdx() > RHS.getStartIdx(); } - /// Compute the registers that are live across this Candidate. - /// Used by targets that need this information for cost model calculation. - /// If a target does not need this information, then this should not be - /// called. - void initLRU(const TargetRegisterInfo &TRI) { - assert(MBB->getParent()->getRegInfo().tracksLiveness() && - "Candidate's Machine Function must track liveness"); - // Only initialize once. - if (LRUWasSet) - return; - LRUWasSet = true; - LRU.init(TRI); - LRU.addLiveOuts(*MBB); - - // Compute liveness from the end of the block up to the beginning of the - // outlining candidate. - std::for_each(MBB->rbegin(), (MachineBasicBlock::reverse_iterator)front(), - [this](MachineInstr &MI) { LRU.stepBackward(MI); }); - - // Walk over the sequence itself and figure out which registers were used - // in the sequence. - UsedInSequence.init(TRI); - std::for_each(front(), std::next(back()), - [this](MachineInstr &MI) { UsedInSequence.accumulate(MI); }); - } }; /// The information necessary to create an outlined function for some diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 75b8a89c812e..6089339c7f5a 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -25,13 +25,15 @@ #include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Error.h" -#include "llvm/Support/type_traits.h" + +#include <map> namespace llvm { class Module; +class Function; +class MachineFunction; extern template class AnalysisManager<MachineFunction>; diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index e6763899a083..7748055f5d35 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -47,6 +47,7 @@ FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ()) FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ()) FUNCTION_PASS("lowerinvoke", LowerInvokePass, ()) FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass, ()) +FUNCTION_PASS("tlshoist", TLSVariableHoistPass, ()) FUNCTION_PASS("verify", VerifierPass, ()) #undef FUNCTION_PASS @@ -119,6 +120,7 @@ DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ()) DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ()) DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) +DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) #undef DUMMY_FUNCTION_PASS #ifndef DUMMY_MODULE_PASS @@ -197,6 +199,5 @@ DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass, ()) DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass, ()) DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass, ()) DUMMY_MACHINE_FUNCTION_PASS("machineverifier", MachineVerifierPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("machine-cycles", MachineCycleInfoWrapperPass, ()) DUMMY_MACHINE_FUNCTION_PASS("print-machine-cycles", MachineCycleInfoPrinterPass, ()) #undef DUMMY_MACHINE_FUNCTION_PASS diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h index 7e7fa57d80da..4559f7a9bde7 100644 --- a/llvm/include/llvm/CodeGen/MachinePipeliner.h +++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h @@ -40,13 +40,17 @@ #ifndef LLVM_CODEGEN_MACHINEPIPELINER_H #define LLVM_CODEGEN_MACHINEPIPELINER_H +#include "llvm/ADT/SetVector.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/CodeGen/ScheduleDAGMutation.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" +#include <deque> + namespace llvm { class AAResults; @@ -80,6 +84,8 @@ public: SmallVector<MachineOperand, 4> BrCond; MachineInstr *LoopInductionVar = nullptr; MachineInstr *LoopCompare = nullptr; + std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> LoopPipelinerInfo = + nullptr; }; LoopInfo LI; @@ -115,6 +121,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs { LiveIntervals &LIS; const RegisterClassInfo &RegClassInfo; unsigned II_setByPragma = 0; + TargetInstrInfo::PipelinerLoopInfo *LoopPipelinerInfo = nullptr; /// A toplogical ordering of the SUnits, which is needed for changing /// dependences and iterating over the SUnits. @@ -192,9 +199,11 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs { public: SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis, - const RegisterClassInfo &rci, unsigned II) + const RegisterClassInfo &rci, unsigned II, + TargetInstrInfo::PipelinerLoopInfo *PLI) : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis), - RegClassInfo(rci), II_setByPragma(II), Topo(SUnits, &ExitSU) { + RegClassInfo(rci), II_setByPragma(II), LoopPipelinerInfo(PLI), + Topo(SUnits, &ExitSU) { P.MF->getSubtarget().getSMSMutations(Mutations); if (SwpEnableCopyToPhi) Mutations.push_back(std::make_unique<CopyToPhiMutation>()); @@ -585,6 +594,13 @@ public: return ScheduledInstrs[cycle]; } + SmallSet<SUnit *, 8> + computeUnpipelineableNodes(SwingSchedulerDAG *SSD, + TargetInstrInfo::PipelinerLoopInfo *PLI); + + bool + normalizeNonPipelinedInstructions(SwingSchedulerDAG *SSD, + TargetInstrInfo::PipelinerLoopInfo *PLI); bool isValidSchedule(SwingSchedulerDAG *SSD); void finalizeSchedule(SwingSchedulerDAG *SSD); void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU, diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h index 94ae6fe02e9c..b2c5f12106af 100644 --- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h @@ -15,18 +15,16 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringSet.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/CodeGen/GlobalISel/RegisterBank.h" -#include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/RegisterBank.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/MC/LaneBitmask.h" @@ -229,6 +227,16 @@ public: /// Returns true if the updated CSR list was initialized and false otherwise. bool isUpdatedCSRsInitialized() const { return IsUpdatedCSRsInitialized; } + /// Returns true if a register can be used as an argument to a function. + bool isArgumentRegister(const MachineFunction &MF, MCRegister Reg) const; + + /// Returns true if a register is a fixed register. + bool isFixedRegister(const MachineFunction &MF, MCRegister Reg) const; + + /// Returns true if a register is a general purpose register. + bool isGeneralPurposeRegister(const MachineFunction &MF, + MCRegister Reg) const; + /// Disables the register from the list of CSRs. /// I.e. the register will not appear as part of the CSR mask. /// \see UpdatedCalleeSavedRegs. @@ -825,23 +833,12 @@ public: /// to refer to the designated register. void updateDbgUsersToReg(MCRegister OldReg, MCRegister NewReg, ArrayRef<MachineInstr *> Users) const { - SmallSet<MCRegister, 4> OldRegUnits; - for (MCRegUnitIterator RUI(OldReg, getTargetRegisterInfo()); RUI.isValid(); - ++RUI) - OldRegUnits.insert(*RUI); - // If this operand is a register, check whether it overlaps with OldReg. // If it does, replace with NewReg. - auto UpdateOp = [this, &NewReg, &OldReg, &OldRegUnits](MachineOperand &Op) { - if (Op.isReg()) { - for (MCRegUnitIterator RUI(OldReg, getTargetRegisterInfo()); - RUI.isValid(); ++RUI) { - if (OldRegUnits.contains(*RUI)) { - Op.setReg(NewReg); - break; - } - } - } + auto UpdateOp = [this, &NewReg, &OldReg](MachineOperand &Op) { + if (Op.isReg() && + getTargetRegisterInfo()->regsOverlap(Op.getReg(), OldReg)) + Op.setReg(NewReg); }; // Iterate through (possibly several) operands to DBG_VALUEs and update diff --git a/llvm/include/llvm/CodeGen/MachineSSAContext.h b/llvm/include/llvm/CodeGen/MachineSSAContext.h index 6dbf321bdeaa..f59d7cf8a522 100644 --- a/llvm/include/llvm/CodeGen/MachineSSAContext.h +++ b/llvm/include/llvm/CodeGen/MachineSSAContext.h @@ -15,21 +15,21 @@ #ifndef LLVM_CODEGEN_MACHINESSACONTEXT_H #define LLVM_CODEGEN_MACHINESSACONTEXT_H -#include "llvm/ADT/GenericSSAContext.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/Support/Printable.h" -#include <memory> - namespace llvm { +class MachineRegisterInfo; class MachineInstr; -class MachineBasicBlock; class MachineFunction; class Register; +template <typename _FunctionT> class GenericSSAContext; template <typename, bool> class DominatorTreeBase; inline auto successors(MachineBasicBlock *BB) { return BB->successors(); } inline auto predecessors(MachineBasicBlock *BB) { return BB->predecessors(); } +inline unsigned succ_size(MachineBasicBlock *BB) { return BB->succ_size(); } +inline unsigned pred_size(MachineBasicBlock *BB) { return BB->pred_size(); } template <> class GenericSSAContext<MachineFunction> { const MachineRegisterInfo *RegInfo = nullptr; diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index 267c4b595eec..0554eb1ab77e 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -287,7 +287,7 @@ protected: const SUnit *NextClusterPred = nullptr; const SUnit *NextClusterSucc = nullptr; -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS /// The number of instructions scheduled so far. Used to cut off the /// scheduler at the point determined by misched-cutoff. unsigned NumInstrsScheduled = 0; @@ -679,7 +679,7 @@ private: // For each PIdx, stores the resource group IDs of its subunits SmallVector<APInt, 16> ResourceGroupSubUnitMasks; -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS // Remember the greatest possible stall as an upper bound on the number of // times we should retry the pending queue because of a hazard. unsigned MaxObservedStall; diff --git a/llvm/include/llvm/CodeGen/MachineStableHash.h b/llvm/include/llvm/CodeGen/MachineStableHash.h index 8423b2da1c78..43571b7b8afd 100644 --- a/llvm/include/llvm/CodeGen/MachineStableHash.h +++ b/llvm/include/llvm/CodeGen/MachineStableHash.h @@ -17,6 +17,8 @@ #include "llvm/CodeGen/StableHashing.h" namespace llvm { +class MachineBasicBlock; +class MachineFunction; class MachineInstr; class MachineOperand; @@ -24,6 +26,8 @@ stable_hash stableHashValue(const MachineOperand &MO); stable_hash stableHashValue(const MachineInstr &MI, bool HashVRegs = false, bool HashConstantPoolIndices = false, bool HashMemOperands = false); +stable_hash stableHashValue(const MachineBasicBlock &MBB); +stable_hash stableHashValue(const MachineFunction &MF); } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h index e8dbf49994bb..c515101e80fd 100644 --- a/llvm/include/llvm/CodeGen/ModuloSchedule.h +++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h @@ -61,7 +61,6 @@ #define LLVM_CODEGEN_MODULOSCHEDULE_H #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineLoopUtils.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -70,6 +69,8 @@ namespace llvm { class MachineBasicBlock; +class MachineLoop; +class MachineRegisterInfo; class MachineInstr; class LiveIntervals; @@ -190,8 +191,8 @@ private: void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB, ValueMapTy *VRMap, MBBVectorTy &PrologBBs); void generateEpilog(unsigned LastStage, MachineBasicBlock *KernelBB, - ValueMapTy *VRMap, MBBVectorTy &EpilogBBs, - MBBVectorTy &PrologBBs); + MachineBasicBlock *OrigBB, ValueMapTy *VRMap, + MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs); void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, ValueMapTy *VRMap, InstrMapTy &InstrMap, diff --git a/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h b/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h index 51822d082bad..043b6b120632 100644 --- a/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h +++ b/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -190,7 +190,7 @@ namespace PBQP { RawVector v = G.getNodeCosts(NId); -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS // Although a conservatively allocatable node can be allocated to a register, // spilling it may provide a lower cost solution. Assert here that spilling // is done by choice, not because there were no register available. diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 616ab1034133..6e37d42f0d29 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -51,10 +51,8 @@ namespace llvm { FunctionPass *createUnreachableBlockEliminationPass(); /// createBasicBlockSections Pass - This pass assigns sections to machine - /// basic blocks and is enabled with -fbasic-block-sections. Buf is a memory - /// buffer that contains the list of functions and basic block ids to - /// selectively enable basic block sections. - MachineFunctionPass *createBasicBlockSectionsPass(const MemoryBuffer *Buf); + /// basic blocks and is enabled with -fbasic-block-sections. + MachineFunctionPass *createBasicBlockSectionsPass(); /// createMachineFunctionSplitterPass - This pass splits machine functions /// using profile information. @@ -331,6 +329,8 @@ namespace llvm { /// machine instructions. extern char &MachineCopyPropagationID; + MachineFunctionPass *createMachineCopyPropagationPass(bool UseCopyInstr); + /// PeepholeOptimizer - This pass performs peephole optimizations - /// like extension and comparison eliminations. extern char &PeepholeOptimizerID; @@ -494,6 +494,9 @@ namespace llvm { // This pass expands indirectbr instructions. FunctionPass *createIndirectBrExpandPass(); + /// Creates CFI Fixup pass. \see CFIFixup.cpp + FunctionPass *createCFIFixup(); + /// Creates CFI Instruction Inserter pass. \see CFIInstrInserter.cpp FunctionPass *createCFIInstrInserter(); @@ -554,6 +557,12 @@ namespace llvm { /// When learning an eviction policy, extract score(reward) information, /// otherwise this does nothing FunctionPass *createRegAllocScoringPass(); + + /// JMC instrument pass. + ModulePass *createJMCInstrumenterPass(); + + /// This pass converts conditional moves to conditional jumps when profitable. + FunctionPass *createSelectOptimizePass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/CodeGen/PseudoSourceValue.h b/llvm/include/llvm/CodeGen/PseudoSourceValue.h index f1487017f205..07b7ba321566 100644 --- a/llvm/include/llvm/CodeGen/PseudoSourceValue.h +++ b/llvm/include/llvm/CodeGen/PseudoSourceValue.h @@ -25,7 +25,7 @@ class MachineMemOperand; class MIRFormatter; class PseudoSourceValue; class raw_ostream; -class TargetInstrInfo; +class TargetMachine; raw_ostream &operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -59,7 +59,7 @@ private: virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); + explicit PseudoSourceValue(unsigned Kind, const TargetMachine &TM); virtual ~PseudoSourceValue(); @@ -95,8 +95,8 @@ class FixedStackPseudoSourceValue : public PseudoSourceValue { const int FI; public: - explicit FixedStackPseudoSourceValue(int FI, const TargetInstrInfo &TII) - : PseudoSourceValue(FixedStack, TII), FI(FI) {} + explicit FixedStackPseudoSourceValue(int FI, const TargetMachine &TM) + : PseudoSourceValue(FixedStack, TM), FI(FI) {} static bool classof(const PseudoSourceValue *V) { return V->kind() == FixedStack; @@ -115,7 +115,7 @@ public: class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); + CallEntryPseudoSourceValue(unsigned Kind, const TargetMachine &TM); public: bool isConstant(const MachineFrameInfo *) const override; @@ -128,8 +128,7 @@ class GlobalValuePseudoSourceValue : public CallEntryPseudoSourceValue { const GlobalValue *GV; public: - GlobalValuePseudoSourceValue(const GlobalValue *GV, - const TargetInstrInfo &TII); + GlobalValuePseudoSourceValue(const GlobalValue *GV, const TargetMachine &TM); static bool classof(const PseudoSourceValue *V) { return V->kind() == GlobalValueCallEntry; @@ -143,7 +142,7 @@ class ExternalSymbolPseudoSourceValue : public CallEntryPseudoSourceValue { const char *ES; public: - ExternalSymbolPseudoSourceValue(const char *ES, const TargetInstrInfo &TII); + ExternalSymbolPseudoSourceValue(const char *ES, const TargetMachine &TM); static bool classof(const PseudoSourceValue *V) { return V->kind() == ExternalSymbolCallEntry; @@ -154,7 +153,7 @@ public: /// Manages creation of pseudo source values. class PseudoSourceValueManager { - const TargetInstrInfo &TII; + const TargetMachine &TM; const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; std::map<int, std::unique_ptr<FixedStackPseudoSourceValue>> FSValues; StringMap<std::unique_ptr<const ExternalSymbolPseudoSourceValue>> @@ -164,7 +163,7 @@ class PseudoSourceValueManager { GlobalCallEntries; public: - PseudoSourceValueManager(const TargetInstrInfo &TII); + PseudoSourceValueManager(const TargetMachine &TM); /// Return a pseudo source value referencing the area below the stack frame of /// a function, e.g., the argument space. diff --git a/llvm/include/llvm/CodeGen/RDFGraph.h b/llvm/include/llvm/CodeGen/RDFGraph.h index e0205d7c92c8..a323ee9dc396 100644 --- a/llvm/include/llvm/CodeGen/RDFGraph.h +++ b/llvm/include/llvm/CodeGen/RDFGraph.h @@ -749,7 +749,6 @@ namespace rdf { RegisterRef makeRegRef(unsigned Reg, unsigned Sub) const; RegisterRef makeRegRef(const MachineOperand &Op) const; - RegisterRef restrictRef(RegisterRef AR, RegisterRef BR) const; NodeAddr<RefNode*> getNextRelated(NodeAddr<InstrNode*> IA, NodeAddr<RefNode*> RA) const; diff --git a/llvm/include/llvm/CodeGen/RegAllocPBQP.h b/llvm/include/llvm/CodeGen/RegAllocPBQP.h index 1ed55082e32c..1ea8840947bc 100644 --- a/llvm/include/llvm/CodeGen/RegAllocPBQP.h +++ b/llvm/include/llvm/CodeGen/RegAllocPBQP.h @@ -183,11 +183,12 @@ public: NodeMetadata() = default; NodeMetadata(const NodeMetadata &Other) - : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), - OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg), - AllowedRegs(Other.AllowedRegs) -#ifndef NDEBUG - , everConservativelyAllocatable(Other.everConservativelyAllocatable) + : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), + OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg), + AllowedRegs(Other.AllowedRegs) +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + , + everConservativelyAllocatable(Other.everConservativelyAllocatable) #endif { if (NumOpts > 0) { @@ -217,7 +218,7 @@ public: assert(RS >= this->RS && "A node's reduction state can not be downgraded"); this->RS = RS; -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS // Remember this state to assert later that a non-infinite register // option was available. if (RS == ConservativelyAllocatable) @@ -247,7 +248,7 @@ public: &OptUnsafeEdges[NumOpts]); } -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS bool wasConservativelyAllocatable() const { return everConservativelyAllocatable; } @@ -261,7 +262,7 @@ private: Register VReg; GraphMetadata::AllowedRegVecRef AllowedRegs; -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS bool everConservativelyAllocatable = false; #endif }; diff --git a/llvm/include/llvm/CodeGen/Register.h b/llvm/include/llvm/CodeGen/Register.h index a683223b5a4a..9dc3e98fe837 100644 --- a/llvm/include/llvm/CodeGen/Register.h +++ b/llvm/include/llvm/CodeGen/Register.h @@ -69,7 +69,7 @@ public: /// Return true if the specified register number is in /// the virtual register namespace. static bool isVirtualRegister(unsigned Reg) { - return Reg & MCRegister::VirtualRegFlag && !isStackSlot(Reg); + return Reg & MCRegister::VirtualRegFlag; } /// Convert a virtual register number to a 0-based index. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/llvm/include/llvm/CodeGen/RegisterBank.h index 5440d97728b4..66885f113e8e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h +++ b/llvm/include/llvm/CodeGen/RegisterBank.h @@ -1,4 +1,4 @@ -//==-- llvm/CodeGen/GlobalISel/RegisterBank.h - Register Bank ----*- C++ -*-==// +//==-- llvm/CodeGen/RegisterBank.h - Register Bank ---------------*- C++ -*-==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_REGISTERBANK_H -#define LLVM_CODEGEN_GLOBALISEL_REGISTERBANK_H +#ifndef LLVM_CODEGEN_REGISTERBANK_H +#define LLVM_CODEGEN_REGISTERBANK_H #include "llvm/ADT/BitVector.h" diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/llvm/include/llvm/CodeGen/RegisterBankInfo.h index da785406bc31..bba4f1f025a0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/llvm/include/llvm/CodeGen/RegisterBankInfo.h @@ -1,4 +1,4 @@ -//===- llvm/CodeGen/GlobalISel/RegisterBankInfo.h ---------------*- C++ -*-===// +//===- llvm/CodeGen/RegisterBankInfo.h --------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_REGISTERBANKINFO_H -#define LLVM_CODEGEN_GLOBALISEL_REGISTERBANKINFO_H +#ifndef LLVM_CODEGEN_REGISTERBANKINFO_H +#define LLVM_CODEGEN_REGISTERBANKINFO_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Hashing.h" @@ -217,8 +217,7 @@ public: const ValueMapping *OperandsMapping, unsigned NumOperands) : ID(ID), Cost(Cost), OperandsMapping(OperandsMapping), - NumOperands(NumOperands) { - } + NumOperands(NumOperands) {} /// Default constructor. /// Use this constructor to express that the mapping is invalid. @@ -630,8 +629,9 @@ public: /// similar to ::copyCost, except for cases where multiple copy-like /// operations need to be inserted. If the register is used as a source /// operand and already has a bank assigned, \p CurBank is non-null. - virtual unsigned getBreakDownCost(const ValueMapping &ValMapping, - const RegisterBank *CurBank = nullptr) const { + virtual unsigned + getBreakDownCost(const ValueMapping &ValMapping, + const RegisterBank *CurBank = nullptr) const { return std::numeric_limits<unsigned>::max(); } @@ -772,4 +772,4 @@ hash_code hash_value(const RegisterBankInfo::PartialMapping &PartMapping); } // end namespace llvm -#endif // LLVM_CODEGEN_GLOBALISEL_REGISTERBANKINFO_H +#endif // LLVM_CODEGEN_REGISTERBANKINFO_H diff --git a/llvm/include/llvm/CodeGen/RegisterClassInfo.h b/llvm/include/llvm/CodeGen/RegisterClassInfo.h index d82f1db60d8b..39c72a42c433 100644 --- a/llvm/include/llvm/CodeGen/RegisterClassInfo.h +++ b/llvm/include/llvm/CodeGen/RegisterClassInfo.h @@ -20,8 +20,7 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/MC/MCRegisterInfo.h" -#include <cassert> +#include "llvm/MC/MCRegister.h" #include <cstdint> #include <memory> @@ -61,6 +60,10 @@ class RegisterClassInfo { // Map register alias to the callee saved Register. SmallVector<MCPhysReg, 4> CalleeSavedAliases; + // Indicate if a specified callee saved register be in the allocation order + // exactly as written in the tablegen descriptions or listed later. + BitVector IgnoreCSRForAllocOrder; + // Reserved registers in the current MF. BitVector Reserved; diff --git a/llvm/include/llvm/CodeGen/RegisterPressure.h b/llvm/include/llvm/CodeGen/RegisterPressure.h index 1deeb4d41511..c40c0eec80ec 100644 --- a/llvm/include/llvm/CodeGen/RegisterPressure.h +++ b/llvm/include/llvm/CodeGen/RegisterPressure.h @@ -22,7 +22,6 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/LaneBitmask.h" #include <cassert> -#include <cstddef> #include <cstdint> #include <cstdlib> #include <limits> diff --git a/llvm/include/llvm/CodeGen/RegisterScavenging.h b/llvm/include/llvm/CodeGen/RegisterScavenging.h index 218e05f6eb6b..1f0cd273bf61 100644 --- a/llvm/include/llvm/CodeGen/RegisterScavenging.h +++ b/llvm/include/llvm/CodeGen/RegisterScavenging.h @@ -70,6 +70,26 @@ class RegScavenger { public: RegScavenger() = default; + /// Record that \p Reg is in use at scavenging index \p FI. This is for + /// targets which need to directly manage the spilling process, and need to + /// update the scavenger's internal state. It's expected this be called a + /// second time with \p Restore set to a non-null value, so that the + /// externally inserted restore instruction resets the scavenged slot + /// liveness when encountered. + void assignRegToScavengingIndex(int FI, Register Reg, + MachineInstr *Restore = nullptr) { + for (ScavengedInfo &Slot : Scavenged) { + if (Slot.FrameIndex == FI) { + assert(!Slot.Reg || Slot.Reg == Reg); + Slot.Reg = Reg; + Slot.Restore = Restore; + return; + } + } + + llvm_unreachable("did not find scavenging index"); + } + /// Start tracking liveness from the begin of basic block \p MBB. void enterBasicBlock(MachineBasicBlock &MBB); diff --git a/llvm/include/llvm/CodeGen/RegisterUsageInfo.h b/llvm/include/llvm/CodeGen/RegisterUsageInfo.h index bf347c0753e5..8b406a275025 100644 --- a/llvm/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/llvm/include/llvm/CodeGen/RegisterUsageInfo.h @@ -20,9 +20,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/IR/Instructions.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" +#include "llvm/PassRegistry.h" #include <cstdint> #include <vector> diff --git a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h index 7c0ebe7191e4..c71aca0c992b 100644 --- a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h +++ b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h @@ -1,4 +1,4 @@ -//===- ReplaceWithVeclib.h - Replace vector instrinsics with veclib calls -===// +//===- ReplaceWithVeclib.h - Replace vector intrinsics with veclib calls --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -17,8 +17,10 @@ #include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" +#include "llvm/PassRegistry.h" namespace llvm { +class Function; struct ReplaceWithVeclib : public PassInfoMixin<ReplaceWithVeclib> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/ScheduleDAG.h b/llvm/include/llvm/CodeGen/ScheduleDAG.h index af8c0cd8756e..f1c377f76d02 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -16,7 +16,6 @@ #define LLVM_CODEGEN_SCHEDULEDAG_H #include "llvm/ADT/BitVector.h" -#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" @@ -31,6 +30,7 @@ namespace llvm { +template <class GraphType> struct GraphTraits; template<class Graph> class GraphWriter; class LLVMTargetMachine; class MachineFunction; diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 50b186de2b05..fb3900b4a9c1 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -16,10 +16,10 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerIntPair.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseMultiSet.h" #include "llvm/ADT/SparseSet.h" +#include "llvm/ADT/identity.h" #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAG.h" diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index e31719bcff0b..bcbd7ebcc0c9 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -20,7 +20,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FoldingSet.h" -#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/ilist.h" @@ -33,17 +32,13 @@ #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/DebugLoc.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" -#include "llvm/Support/AtomicOrdering.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/RecyclingAllocator.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <functional> @@ -55,6 +50,15 @@ namespace llvm { +class DIExpression; +class DILabel; +class DIVariable; +class Function; +class Pass; +class Type; +template <class GraphType> struct GraphTraits; +template <typename T, unsigned int N> class SmallSetVector; +template <typename T, typename Enable> struct FoldingSetTrait; class AAResults; class BlockAddress; class BlockFrequencyInfo; @@ -276,8 +280,16 @@ class SelectionDAG { DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo; + /// PersistentId counter to be used when inserting the next + /// SDNode to this SelectionDAG. We do not place that under + /// `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` intentionally because + /// it adds unneeded complexity without noticeable + /// benefits (see discussion with @thakis in D120714). uint16_t NextPersistentId = 0; + /// Are instruction referencing variable locations desired for this function? + bool UseInstrRefDebugInfo = false; + public: /// Clients of various APIs that cause global effects on /// the DAG can optionally implement this interface. This allows the clients @@ -440,6 +452,9 @@ public: const DataLayout &getDataLayout() const { return MF->getDataLayout(); } const TargetMachine &getTarget() const { return TM; } const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } + template <typename STC> const STC &getSubtarget() const { + return MF->getSubtarget<STC>(); + } const TargetLowering &getTargetLoweringInfo() const { return *TLI; } const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } @@ -467,7 +482,7 @@ public: void viewGraph(const std::string &Title); void viewGraph(); -#ifndef NDEBUG +#if LLVM_ENABLE_ABI_BREAKING_CHECKS std::map<const SDNode *, std::string> NodeGraphAttrs; #endif @@ -893,6 +908,11 @@ public: /// Create a logical NOT operation as (XOR Val, BooleanOne). SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); + /// Create a vector-predicated logical NOT operation as (VP_XOR Val, + /// BooleanOne, Mask, EVL). + SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, + SDValue EVL, EVT VT); + /// Returns sum of the base pointer and offset. /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, @@ -1032,25 +1052,26 @@ public: const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, - SDValue Size, Align Alignment, bool isVol, bool isTailCall, + SDValue Size, Align Alignment, bool isVol, + bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo = AAMDNodes()); SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, - unsigned DstAlign, SDValue Src, unsigned SrcAlign, - SDValue Size, Type *SizeTy, unsigned ElemSz, - bool isTailCall, MachinePointerInfo DstPtrInfo, + SDValue Src, SDValue Size, Type *SizeTy, + unsigned ElemSz, bool isTailCall, + MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo); SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, - unsigned DstAlign, SDValue Src, unsigned SrcAlign, - SDValue Size, Type *SizeTy, unsigned ElemSz, - bool isTailCall, MachinePointerInfo DstPtrInfo, + SDValue Src, SDValue Size, Type *SizeTy, + unsigned ElemSz, bool isTailCall, + MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo); SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, - unsigned DstAlign, SDValue Value, SDValue Size, - Type *SizeTy, unsigned ElemSz, bool isTailCall, + SDValue Value, SDValue Size, Type *SizeTy, + unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo); /// Helper function to make it easier to build SetCC's if you just have an @@ -1070,14 +1091,24 @@ public: return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); } + /// Helper function to make it easier to build VP_SETCCs if you just have an + /// ISD::CondCode instead of an SDValue. + SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, + ISD::CondCode Cond, SDValue Mask, SDValue EVL) { + assert(LHS.getValueType().isVector() && RHS.getValueType().isVector() && + "Cannot compare scalars"); + assert(Cond != ISD::SETCC_INVALID && + "Cannot create a setCC of an invalid node."); + return getNode(ISD::VP_SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Mask, + EVL); + } + /// Helper function to make it easier to build Select's if you just have /// operands and don't want to check for vector. SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS) { - assert(LHS.getValueType() == RHS.getValueType() && + assert(LHS.getValueType() == VT && RHS.getValueType() == VT && "Cannot use select on differing types"); - assert(VT.isVector() == LHS.getValueType().isVector() && - "Cannot mix vectors and scalars"); auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; return getNode(Opcode, DL, VT, Cond, LHS, RHS); } @@ -1149,7 +1180,7 @@ public: uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) { // Ensure that codegen never sees alignment 0 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, - Alignment.getValueOr(getEVTAlign(MemVT)), Flags, + Alignment.value_or(getEVTAlign(MemVT)), Flags, Size, AAInfo); } @@ -1230,7 +1261,7 @@ public: const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) { // Ensures that codegen never sees a None Alignment. return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, - Alignment.getValueOr(getEVTAlign(MemVT)), MMOFlags, AAInfo, + Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo, Ranges); } /// FIXME: Remove once transition to Align is over. @@ -1264,7 +1295,7 @@ public: MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, const AAMDNodes &AAInfo = AAMDNodes()) { return getStore(Chain, dl, Val, Ptr, PtrInfo, - Alignment.getValueOr(getEVTAlign(Val.getValueType())), + Alignment.value_or(getEVTAlign(Val.getValueType())), MMOFlags, AAInfo); } /// FIXME: Remove once transition to Align is over. @@ -1290,7 +1321,7 @@ public: MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, const AAMDNodes &AAInfo = AAMDNodes()) { return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, - Alignment.getValueOr(getEVTAlign(SVT)), MMOFlags, + Alignment.value_or(getEVTAlign(SVT)), MMOFlags, AAInfo); } /// FIXME: Remove once transition to Align is over. @@ -1323,7 +1354,7 @@ public: const MDNode *Ranges = nullptr, bool IsExpanding = false) { // Ensures that codegen never sees a None Alignment. return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, - PtrInfo, MemVT, Alignment.getValueOr(getEVTAlign(MemVT)), + PtrInfo, MemVT, Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo, Ranges, IsExpanding); } SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, @@ -1364,6 +1395,77 @@ public: SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM); + SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, + EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, + SDValue Offset, SDValue Stride, SDValue Mask, + SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, + Align Alignment, MachineMemOperand::Flags MMOFlags, + const AAMDNodes &AAInfo, + const MDNode *Ranges = nullptr, + bool IsExpanding = false); + inline SDValue getStridedLoadVP( + ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, + SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, + SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, + MaybeAlign Alignment = MaybeAlign(), + MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, + const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, + bool IsExpanding = false) { + // Ensures that codegen never sees a None Alignment. + return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, + Mask, EVL, PtrInfo, MemVT, + Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, + AAInfo, Ranges, IsExpanding); + } + SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, + EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, + SDValue Offset, SDValue Stride, SDValue Mask, + SDValue EVL, EVT MemVT, MachineMemOperand *MMO, + bool IsExpanding = false); + SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, + SDValue Stride, SDValue Mask, SDValue EVL, + MachinePointerInfo PtrInfo, MaybeAlign Alignment, + MachineMemOperand::Flags MMOFlags, + const AAMDNodes &AAInfo, + const MDNode *Ranges = nullptr, + bool IsExpanding = false); + SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, + SDValue Stride, SDValue Mask, SDValue EVL, + MachineMemOperand *MMO, bool IsExpanding = false); + SDValue + getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, + SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, + SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, + MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, + const AAMDNodes &AAInfo, bool IsExpanding = false); + SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, + SDValue Chain, SDValue Ptr, SDValue Stride, + SDValue Mask, SDValue EVL, EVT MemVT, + MachineMemOperand *MMO, bool IsExpanding = false); + SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, + SDValue Base, SDValue Offset, + ISD::MemIndexedMode AM); + SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, + SDValue Ptr, SDValue Offset, SDValue Stride, + SDValue Mask, SDValue EVL, EVT MemVT, + MachineMemOperand *MMO, ISD::MemIndexedMode AM, + bool IsTruncating = false, + bool IsCompressing = false); + SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, + SDValue Ptr, SDValue Stride, SDValue Mask, + SDValue EVL, MachinePointerInfo PtrInfo, + EVT SVT, Align Alignment, + MachineMemOperand::Flags MMOFlags, + const AAMDNodes &AAInfo, + bool IsCompressing = false); + SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, + SDValue Ptr, SDValue Stride, SDValue Mask, + SDValue EVL, EVT SVT, MachineMemOperand *MMO, + bool IsCompressing = false); + SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, + SDValue Base, SDValue Offset, + ISD::MemIndexedMode AM); + SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType); @@ -1412,6 +1514,11 @@ public: /// Return an AssertAlignSDNode. SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A); + /// Swap N1 and N2 if Opcode is a commutative binary opcode + /// and the canonical form expects the opposite order. + void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, + SDValue &N2) const; + /// Return the specified value casted to /// the target's desired shift amount type. SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); @@ -1702,6 +1809,16 @@ public: /// function mirrors \c llvm::salvageDebugInfo. void salvageDebugInfo(SDNode &N); + /// Signal whether instruction referencing variable locations are desired for + /// this function's debug-info. + void useInstrRefDebugInfo(bool Flag) { + UseInstrRefDebugInfo = Flag; + } + + bool getUseInstrRefDebugInfo() const { + return UseInstrRefDebugInfo; + } + void dump() const; /// In most cases this function returns the ABI alignment for a given type, @@ -1745,16 +1862,6 @@ public: /// simplify nodes with multiple uses more aggressively.) SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits); - /// See if the specified operand can be simplified with the knowledge that - /// only the bits specified by DemandedBits are used in the elements specified - /// by DemandedElts. If so, return the simpler operand, otherwise return a - /// null SDValue. - /// - /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can - /// simplify nodes with multiple uses more aggressively.) - SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits, - const APInt &DemandedElts); - /// Return true if the sign bit of Op is known to be zero. /// We use this predicate to simplify operations downstream. bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; @@ -1771,6 +1878,11 @@ public: bool MaskedValueIsZero(SDValue Op, const APInt &Mask, const APInt &DemandedElts, unsigned Depth = 0) const; + /// Return true if 'Op' is known to be zero in DemandedElts. We + /// use this predicate to simplify operations downstream. + bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, + unsigned Depth = 0) const; + /// Return true if '(Op & Mask) == Mask'. /// Op and Mask are known to be the same type. bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, @@ -2020,11 +2132,6 @@ public: /// Compute the default alignment value for the given type. Align getEVTAlign(EVT MemoryVT) const; - /// Compute the default alignment value for the given type. - /// FIXME: Remove once transition to Align is over. - inline unsigned getEVTAlignment(EVT MemoryVT) const { - return getEVTAlign(MemoryVT).value(); - } /// Test whether the given value is a constant int or similar node. SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const; @@ -2039,39 +2146,34 @@ public: isConstantFPBuildVectorOrConstantFP(N); } - void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { - SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo); + /// Set CallSiteInfo to be associated with Node. + void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo) { + SDCallSiteDbgInfo[Node].CSInfo = std::move(CallInfo); } - - CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) { - auto I = SDCallSiteDbgInfo.find(CallNode); - if (I != SDCallSiteDbgInfo.end()) - return std::move(I->second).CSInfo; - return CallSiteInfo(); + /// Return CallSiteInfo associated with Node, or a default if none exists. + CallSiteInfo getCallSiteInfo(const SDNode *Node) { + auto I = SDCallSiteDbgInfo.find(Node); + return I != SDCallSiteDbgInfo.end() ? std::move(I->second).CSInfo + : CallSiteInfo(); } - + /// Set HeapAllocSite to be associated with Node. void addHeapAllocSite(const SDNode *Node, MDNode *MD) { SDCallSiteDbgInfo[Node].HeapAllocSite = MD; } - - /// Return the HeapAllocSite type associated with the SDNode, if it exists. - MDNode *getHeapAllocSite(const SDNode *Node) { - auto It = SDCallSiteDbgInfo.find(Node); - if (It == SDCallSiteDbgInfo.end()) - return nullptr; - return It->second.HeapAllocSite; + /// Return HeapAllocSite associated with Node, or nullptr if none exists. + MDNode *getHeapAllocSite(const SDNode *Node) const { + auto I = SDCallSiteDbgInfo.find(Node); + return I != SDCallSiteDbgInfo.end() ? I->second.HeapAllocSite : nullptr; } - + /// Set NoMergeSiteInfo to be associated with Node if NoMerge is true. void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { if (NoMerge) SDCallSiteDbgInfo[Node].NoMerge = NoMerge; } - - bool getNoMergeSiteInfo(const SDNode *Node) { + /// Return NoMerge info associated with Node. + bool getNoMergeSiteInfo(const SDNode *Node) const { auto I = SDCallSiteDbgInfo.find(Node); - if (I == SDCallSiteDbgInfo.end()) - return false; - return I->second.NoMerge; + return I != SDCallSiteDbgInfo.end() ? I->second.NoMerge : false; } /// Return the current function's default denormal handling kind for the given diff --git a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 0f3af915da64..e23eebec81db 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -49,7 +49,7 @@ public: SDValue getBase() const { return Base; } SDValue getIndex() { return Index; } SDValue getIndex() const { return Index; } - bool hasValidOffset() const { return Offset.hasValue(); } + bool hasValidOffset() const { return Offset.has_value(); } int64_t getOffset() const { return *Offset; } // Returns true if `Other` and `*this` are both some offset from the same base diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 9cea197724cc..35fb0bc80593 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -16,12 +16,13 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/BasicBlock.h" #include <memory> namespace llvm { class AAResults; +class TargetInstrInfo; +class TargetMachine; class SelectionDAGBuilder; class SDValue; class MachineRegisterInfo; @@ -53,6 +54,7 @@ public: const TargetLowering *TLI; bool FastISelFailed; SmallPtrSet<const Instruction *, 4> ElidedArgCopyInstrs; + bool UseInstrRefDebugInfo = false; /// Current optimization remark emitter. /// Used to report things like combines and FastISel failures. diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 04c6b50197d4..5974f13a296b 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -508,7 +508,7 @@ BEGIN_TWO_BYTE_PACK() class LSBaseSDNodeBitfields { friend class LSBaseSDNode; - friend class VPLoadStoreSDNode; + friend class VPBaseLoadStoreSDNode; friend class MaskedLoadStoreSDNode; friend class MaskedGatherScatterSDNode; friend class VPGatherScatterSDNode; @@ -529,6 +529,7 @@ BEGIN_TWO_BYTE_PACK() class LoadSDNodeBitfields { friend class LoadSDNode; friend class VPLoadSDNode; + friend class VPStridedLoadSDNode; friend class MaskedLoadSDNode; friend class MaskedGatherSDNode; friend class VPGatherSDNode; @@ -542,6 +543,7 @@ BEGIN_TWO_BYTE_PACK() class StoreSDNodeBitfields { friend class StoreSDNode; friend class VPStoreSDNode; + friend class VPStridedStoreSDNode; friend class MaskedStoreSDNode; friend class MaskedScatterSDNode; friend class VPScatterSDNode; @@ -613,8 +615,10 @@ private: SDNodeFlags Flags; public: - /// Unique and persistent id per SDNode in the DAG. - /// Used for debug printing. + /// Unique and persistent id per SDNode in the DAG. Used for debug printing. + /// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` + /// intentionally because it adds unneeded complexity without noticeable + /// benefits (see discussion with @thakis in D120714). uint16_t PersistentId; //===--------------------------------------------------------------------===// @@ -1191,12 +1195,13 @@ inline void SDValue::dumpr(const SelectionDAG *G) const { inline void SDUse::set(const SDValue &V) { if (Val.getNode()) removeFromList(); Val = V; - if (V.getNode()) V.getNode()->addUse(*this); + if (V.getNode()) + V->addUse(*this); } inline void SDUse::setInitial(const SDValue &V) { Val = V; - V.getNode()->addUse(*this); + V->addUse(*this); } inline void SDUse::setNode(SDNode *N) { @@ -1364,6 +1369,7 @@ public: case ISD::VP_STORE: case ISD::MSTORE: case ISD::VP_SCATTER: + case ISD::EXPERIMENTAL_VP_STRIDED_STORE: return getOperand(2); case ISD::MGATHER: case ISD::MSCATTER: @@ -1407,6 +1413,8 @@ public: case ISD::VP_STORE: case ISD::VP_GATHER: case ISD::VP_SCATTER: + case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: + case ISD::EXPERIMENTAL_VP_STRIDED_STORE: return true; default: return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); @@ -1661,6 +1669,9 @@ bool isAllOnesConstant(SDValue V); /// Returns true if \p V is a constant integer one. bool isOneConstant(SDValue V); +/// Returns true if \p V is a constant min signed integer value. +bool isMinSignedConstant(SDValue V); + /// Return the non-bitcasted source operand of \p V if it exists. /// If \p V is not a bitcasted value, it is returned as-is. SDValue peekThroughBitcasts(SDValue V); @@ -1677,6 +1688,11 @@ SDValue peekThroughExtractSubvectors(SDValue V); /// constant is canonicalized to be operand 1. bool isBitwiseNot(SDValue V, bool AllowUndefs = false); +/// If \p V is a bitwise not, returns the inverted operand. Otherwise returns +/// an empty SDValue. Only bits set in \p Mask are required to be inverted, +/// other bits may be arbitrary. +SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs); + /// Returns the SDNode if it is a constant splat BuildVector or constant int. ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, bool AllowTruncation = false); @@ -2353,34 +2369,64 @@ public: } }; -/// This base class is used to represent VP_LOAD and VP_STORE nodes -class VPLoadStoreSDNode : public MemSDNode { +/// This base class is used to represent VP_LOAD, VP_STORE, +/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes +class VPBaseLoadStoreSDNode : public MemSDNode { public: friend class SelectionDAG; - VPLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, - SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, - MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, + const DebugLoc &DL, SDVTList VTs, + ISD::MemIndexedMode AM, EVT MemVT, + MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) { LSBaseSDNodeBits.AddressingMode = AM; assert(getAddressingMode() == AM && "Value truncated"); } - // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL) - // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL) + // VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL) + // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL) + // VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL) + // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL) // Mask is a vector of i1 elements; // the type of EVL is TLI.getVPExplicitVectorLengthTy(). const SDValue &getOffset() const { - return getOperand(getOpcode() == ISD::VP_LOAD ? 2 : 3); + return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || + getOpcode() == ISD::VP_LOAD) + ? 2 + : 3); } const SDValue &getBasePtr() const { - return getOperand(getOpcode() == ISD::VP_LOAD ? 1 : 2); + return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || + getOpcode() == ISD::VP_LOAD) + ? 1 + : 2); } const SDValue &getMask() const { - return getOperand(getOpcode() == ISD::VP_LOAD ? 3 : 4); + switch (getOpcode()) { + default: + llvm_unreachable("Invalid opcode"); + case ISD::VP_LOAD: + return getOperand(3); + case ISD::VP_STORE: + case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: + return getOperand(4); + case ISD::EXPERIMENTAL_VP_STRIDED_STORE: + return getOperand(5); + } } const SDValue &getVectorLength() const { - return getOperand(getOpcode() == ISD::VP_LOAD ? 4 : 5); + switch (getOpcode()) { + default: + llvm_unreachable("Invalid opcode"); + case ISD::VP_LOAD: + return getOperand(4); + case ISD::VP_STORE: + case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: + return getOperand(5); + case ISD::EXPERIMENTAL_VP_STRIDED_STORE: + return getOperand(6); + } } /// Return the addressing mode for this load or store: @@ -2396,19 +2442,21 @@ public: bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } static bool classof(const SDNode *N) { - return N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE; + return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || + N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE || + N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE; } }; /// This class is used to represent a VP_LOAD node -class VPLoadSDNode : public VPLoadStoreSDNode { +class VPLoadSDNode : public VPBaseLoadStoreSDNode { public: friend class SelectionDAG; VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding, EVT MemVT, MachineMemOperand *MMO) - : VPLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) { + : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) { LoadSDNodeBits.ExtTy = ETy; LoadSDNodeBits.IsExpanding = isExpanding; } @@ -2428,15 +2476,45 @@ public: bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } }; +/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node. +class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode { +public: + friend class SelectionDAG; + + VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, + ISD::MemIndexedMode AM, ISD::LoadExtType ETy, + bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) + : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs, + AM, MemVT, MMO) { + LoadSDNodeBits.ExtTy = ETy; + LoadSDNodeBits.IsExpanding = IsExpanding; + } + + ISD::LoadExtType getExtensionType() const { + return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); + } + + const SDValue &getBasePtr() const { return getOperand(1); } + const SDValue &getOffset() const { return getOperand(2); } + const SDValue &getStride() const { return getOperand(3); } + const SDValue &getMask() const { return getOperand(4); } + const SDValue &getVectorLength() const { return getOperand(5); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD; + } + bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } +}; + /// This class is used to represent a VP_STORE node -class VPStoreSDNode : public VPLoadStoreSDNode { +class VPStoreSDNode : public VPBaseLoadStoreSDNode { public: friend class SelectionDAG; VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, EVT MemVT, MachineMemOperand *MMO) - : VPLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) { + : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) { StoreSDNodeBits.IsTruncating = isTrunc; StoreSDNodeBits.IsCompressing = isCompressing; } @@ -2463,6 +2541,43 @@ public: } }; +/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node. +class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode { +public: + friend class SelectionDAG; + + VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, + ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing, + EVT MemVT, MachineMemOperand *MMO) + : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL, + VTs, AM, MemVT, MMO) { + StoreSDNodeBits.IsTruncating = IsTrunc; + StoreSDNodeBits.IsCompressing = IsCompressing; + } + + /// Return true if this is a truncating store. + /// For integers this is the same as doing a TRUNCATE and storing the result. + /// For floats, it is the same as doing an FP_ROUND and storing the result. + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } + + /// Returns true if the op does a compression to the vector before storing. + /// The node contiguously stores the active elements (integers or floats) + /// in src (those with their respective bit set in writemask k) to unaligned + /// memory at base_addr. + bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } + + const SDValue &getValue() const { return getOperand(1); } + const SDValue &getBasePtr() const { return getOperand(2); } + const SDValue &getOffset() const { return getOperand(3); } + const SDValue &getStride() const { return getOperand(4); } + const SDValue &getMask() const { return getOperand(5); } + const SDValue &getVectorLength() const { return getOperand(6); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE; + } +}; + /// This base class is used to represent MLOAD and MSTORE nodes class MaskedLoadStoreSDNode : public MemSDNode { public: @@ -2588,13 +2703,9 @@ public: return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); } bool isIndexScaled() const { - return (getIndexType() == ISD::SIGNED_SCALED) || - (getIndexType() == ISD::UNSIGNED_SCALED); - } - bool isIndexSigned() const { - return (getIndexType() == ISD::SIGNED_SCALED) || - (getIndexType() == ISD::SIGNED_UNSCALED); + return !cast<ConstantSDNode>(getScale())->isOne(); } + bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); } // In the both nodes address is Op1, mask is Op2: // VPGatherSDNode (Chain, base, index, scale, mask, vlen) @@ -2675,17 +2786,10 @@ public: ISD::MemIndexType getIndexType() const { return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); } - void setIndexType(ISD::MemIndexType IndexType) { - LSBaseSDNodeBits.AddressingMode = IndexType; - } bool isIndexScaled() const { - return (getIndexType() == ISD::SIGNED_SCALED) || - (getIndexType() == ISD::UNSIGNED_SCALED); - } - bool isIndexSigned() const { - return (getIndexType() == ISD::SIGNED_SCALED) || - (getIndexType() == ISD::SIGNED_UNSCALED); + return !cast<ConstantSDNode>(getScale())->isOne(); } + bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); } // In the both nodes address is Op1, mask is Op2: // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) diff --git a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h index 722c3275fd06..e7d608969124 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h @@ -76,11 +76,13 @@ public: /// that don't fit the target's parameters for simple stores and can be more /// efficient than using a library call. This function can return a null /// SDValue if the target declines to use custom code and a different - /// lowering strategy should be used. + /// lowering strategy should be used. Note that if AlwaysInline is true the + /// function has to return a valid SDValue. virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, + bool AlwaysInline, MachinePointerInfo DstPtrInfo) const { return SDValue(); } diff --git a/llvm/include/llvm/CodeGen/SlotIndexes.h b/llvm/include/llvm/CodeGen/SlotIndexes.h index e8d618a24f9b..942a47c6cc7d 100644 --- a/llvm/include/llvm/CodeGen/SlotIndexes.h +++ b/llvm/include/llvm/CodeGen/SlotIndexes.h @@ -28,7 +28,6 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBundle.h" -#include "llvm/Pass.h" #include "llvm/Support/Allocator.h" #include <algorithm> #include <cassert> diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h index 928d7cc6cc04..01cc9bc37931 100644 --- a/llvm/include/llvm/CodeGen/StackMaps.h +++ b/llvm/include/llvm/CodeGen/StackMaps.h @@ -13,7 +13,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/IR/CallingConv.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/Support/Debug.h" #include <algorithm> #include <cassert> @@ -23,6 +22,7 @@ namespace llvm { class AsmPrinter; +class MCSymbol; class MCExpr; class MCStreamer; class raw_ostream; diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h index 57456b3f6c16..b96c0c74fabc 100644 --- a/llvm/include/llvm/CodeGen/StackProtector.h +++ b/llvm/include/llvm/CodeGen/StackProtector.h @@ -20,7 +20,6 @@ #include "llvm/ADT/Triple.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/ValueMap.h" #include "llvm/Pass.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h b/llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h index 08ab2abbdd5b..a374736347f6 100644 --- a/llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h +++ b/llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h @@ -20,8 +20,6 @@ #include "llvm/CodeGen/Register.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DebugLoc.h" -#include <functional> -#include <type_traits> #include <utility> diff --git a/llvm/include/llvm/CodeGen/TailDuplicator.h b/llvm/include/llvm/CodeGen/TailDuplicator.h index daaa27f72d52..94e8092319d7 100644 --- a/llvm/include/llvm/CodeGen/TailDuplicator.h +++ b/llvm/include/llvm/CodeGen/TailDuplicator.h @@ -16,15 +16,16 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/MBFIWrapper.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include <utility> #include <vector> namespace llvm { +template <typename T, unsigned int N> class SmallSetVector; +template <typename Fn> class function_ref; +class MBFIWrapper; class MachineBasicBlock; class MachineBranchProbabilityInfo; class MachineFunction; diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h index 62365330379d..1333f2d98973 100644 --- a/llvm/include/llvm/CodeGen/TargetCallingConv.h +++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -46,7 +46,8 @@ namespace ISD { unsigned IsHvaStart : 1; ///< HVA structure start unsigned IsSecArgPass : 1; ///< Second argument unsigned MemAlign : 4; ///< Log 2 of alignment when arg is passed in memory - ///< (including byval/byref) + ///< (including byval/byref). The max alignment is + ///< verified in IR verification. unsigned OrigAlign : 5; ///< Log 2 of original alignment unsigned IsInConsecutiveRegsLast : 1; unsigned IsInConsecutiveRegs : 1; diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h index f2ca1590fc39..fbce5d7a9102 100644 --- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h +++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h @@ -213,12 +213,24 @@ public: virtual void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const = 0; + /// emitZeroCallUsedRegs - Zeros out call used registers. + virtual void emitZeroCallUsedRegs(BitVector RegsToZero, + MachineBasicBlock &MBB) const {} + /// With basic block sections, emit callee saved frame moves for basic blocks /// that are in a different section. virtual void emitCalleeSavedFrameMovesFullCFA(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {} + /// Returns true if we may need to fix the unwind information for the + /// function. + virtual bool enableCFIFixup(MachineFunction &MF) const; + + /// Emit CFI instructions that recreate the state of the unwind information + /// upon fucntion entry. + virtual void resetCFIToInitialState(MachineBasicBlock &MBB) const {} + /// Replace a StackProbe stub (if any) with the actual probe code inline virtual void inlineStackProbe(MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {} diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 411811d08c18..f9183e0a9c66 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -382,6 +382,17 @@ public: /// to which instructions should be sunk. virtual bool shouldSink(const MachineInstr &MI) const { return true; } + /// Return false if the instruction should not be hoisted by MachineLICM. + /// + /// MachineLICM determines on its own whether the instruction is safe to + /// hoist; this gives the target a hook to extend this assessment and prevent + /// an instruction being hoisted from a given loop for target specific + /// reasons. + virtual bool shouldHoist(const MachineInstr &MI, + const MachineLoop *FromLoop) const { + return true; + } + /// Re-issue the specified 'original' instruction at the /// specific location targeting a new destination register. /// The register in Orig->getOperand(0).getReg() will be substituted by @@ -723,12 +734,16 @@ public: virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const = 0; /// Create a condition to determine if the trip count of the loop is greater - /// than TC. + /// than TC, where TC is always one more than for the previous prologue or + /// 0 if this is being called for the outermost prologue. /// /// If the trip count is statically known to be greater than TC, return /// true. If the trip count is statically known to be not greater than TC, /// return false. Otherwise return nullopt and fill out Cond with the test /// condition. + /// + /// Note: This hook is guaranteed to be called from the innermost to the + /// outermost prologue of the loop being software pipelined. virtual Optional<bool> createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond) = 0; @@ -1268,13 +1283,6 @@ protected: } public: - /// getAddressSpaceForPseudoSourceKind - Given the kind of memory - /// (e.g. stack) the target returns the corresponding address space. - virtual unsigned - getAddressSpaceForPseudoSourceKind(unsigned Kind) const { - return 0; - } - /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. @@ -1942,7 +1950,7 @@ public: virtual MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const { + outliner::Candidate &C) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::insertOutlinedCall!"); } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 3861648a5feb..98b9a416ea59 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -25,7 +25,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLArrayExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/DAGCombine.h" @@ -248,12 +248,21 @@ public: /// w.r.t. what they should expand to. enum class AtomicExpansionKind { None, // Don't expand the instruction. + CastToInteger, // Cast the atomic instruction to another type, e.g. from + // floating-point to integer type. LLSC, // Expand the instruction into loadlinked/storeconditional; used // by ARM/AArch64. LLOnly, // Expand the (load) instruction into just a load-linked, which has // greater atomic guarantees than a normal load. CmpXChg, // Expand the instruction into cmpxchg; used by at least X86. - MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop. + MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop. + BitTestIntrinsic, // Use a target-specific intrinsic for special bit + // operations; used by X86. + Expand, // Generic expansion in terms of other atomic operations. + + // Rewrite to a non-atomic form for use in a known non-preemptible + // environment. + NotAtomic }; /// Enum that specifies when a multiplication should be expanded. @@ -1071,6 +1080,11 @@ public: return false; } + /// How to legalize this custom operation? + virtual LegalizeAction getCustomOperationAction(SDNode &Op) const { + return Legal; + } + /// Return how this operation should be treated: either it is legal, needs to /// be promoted to a larger size, needs to be expanded to some other code /// sequence, or the target has a custom expander for it. @@ -1210,6 +1224,10 @@ public: uint64_t Range, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const; + /// Returns preferred type for switch condition. + virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, + EVT ConditionVT) const; + /// Return true if lowering to a bit test is suitable for a set of case /// clusters which contains \p NumDests unique destinations, \p Low and /// \p High as its lowest and highest case values, and expects \p NumCmps @@ -1372,7 +1390,9 @@ public: // Returns true if VT is a legal index type for masked gathers/scatters // on this target - virtual bool shouldRemoveExtendFromGSIndex(EVT VT) const { return false; } + virtual bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const { + return false; + } /// Return how the condition code should be treated: either it is legal, needs /// to be expanded to some other code sequence, or the target has a custom @@ -1871,7 +1891,7 @@ public: /// minimum size the object must be to be aligned and PrefAlign is set to the /// preferred alignment. virtual bool shouldAlignPointerArgs(CallInst * /*CI*/, unsigned & /*MinSize*/, - unsigned & /*PrefAlign*/) const { + Align & /*PrefAlign*/) const { return false; } @@ -1946,6 +1966,14 @@ public: llvm_unreachable("Masked atomicrmw expansion unimplemented on this target"); } + /// Perform a bit test atomicrmw using a target-specific intrinsic. This + /// represents the combined bit test intrinsic which will be lowered at a late + /// stage by the backend. + virtual void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const { + llvm_unreachable( + "Bit test atomicrmw expansion unimplemented on this target"); + } + /// Perform a masked cmpxchg using a target-specific intrinsic. This /// represents the core LL/SC loop which will be lowered at a late stage by /// the backend. @@ -2005,12 +2033,6 @@ public: // be unnecessarily held, except if clrex, inserted by this hook, is executed. virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const {} - /// Returns true if the given (atomic) store should be expanded by the - /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input. - virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const { - return false; - } - /// Returns true if arguments should be sign-extended in lib calls. virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const { return IsSigned; @@ -2027,6 +2049,30 @@ public: return AtomicExpansionKind::None; } + /// Returns how the given (atomic) load should be cast by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const { + if (LI->getType()->isFloatingPointTy()) + return AtomicExpansionKind::CastToInteger; + return AtomicExpansionKind::None; + } + + /// Returns how the given (atomic) store should be expanded by the IR-level + /// AtomicExpand pass into. For instance AtomicExpansionKind::Expand will try + /// to use an atomicrmw xchg. + virtual AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const { + return AtomicExpansionKind::None; + } + + /// Returns how the given (atomic) store should be cast by the IR-level + /// AtomicExpand pass into. For instance AtomicExpansionKind::CastToInteger + /// will try to cast the operands to integer values. + virtual AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const { + if (SI->getValueOperand()->getType()->isFloatingPointTy()) + return AtomicExpansionKind::CastToInteger; + return AtomicExpansionKind::None; + } + /// Returns how the given atomic cmpxchg should be expanded by the IR-level /// AtomicExpand pass. virtual AtomicExpansionKind @@ -2041,6 +2087,18 @@ public: AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None; } + /// Returns how the given atomic atomicrmw should be cast by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind + shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const { + if (RMWI->getOperation() == AtomicRMWInst::Xchg && + (RMWI->getValOperand()->getType()->isFloatingPointTy() || + RMWI->getValOperand()->getType()->isPointerTy())) + return AtomicExpansionKind::CastToInteger; + + return AtomicExpansionKind::None; + } + /// On some platforms, an AtomicRMW that never actually modifies the value /// (such as fetch_add of 0) can be turned into a fence followed by an /// atomic load. This may sound useless, but it makes it possible for the @@ -2123,8 +2181,8 @@ public: /// about some cases, a default true can be returned to let the DAGCombiner /// decide. /// AddNode is (add x, c1), and ConstNode is c2. - virtual bool isMulAddWithConstProfitable(const SDValue &AddNode, - const SDValue &ConstNode) const { + virtual bool isMulAddWithConstProfitable(SDValue AddNode, + SDValue ConstNode) const { return true; } @@ -2138,6 +2196,18 @@ public: return false; } + /// Return true if it is beneficial to expand an @llvm.powi.* intrinsic. + /// If not optimizing for size, expanding @llvm.powi.* intrinsics is always + /// considered beneficial. + /// If optimizing for size, expansion is only considered beneficial for upto + /// 5 multiplies and a divide (if the exponent is negative). + bool isBeneficialToExpandPowI(int Exponent, bool OptForSize) const { + if (Exponent < 0) + Exponent = -Exponent; + return !OptForSize || + (countPopulation((unsigned int)Exponent) + Log2_32(Exponent) < 7); + } + //===--------------------------------------------------------------------===// // TargetLowering Configuration Methods - These methods should be invoked by // the derived class constructor to configure this object for the target. @@ -2232,6 +2302,16 @@ protected: assert(Op < array_lengthof(OpActions[0]) && "Table isn't big enough!"); OpActions[(unsigned)VT.SimpleTy][Op] = Action; } + void setOperationAction(ArrayRef<unsigned> Ops, MVT VT, + LegalizeAction Action) { + for (auto Op : Ops) + setOperationAction(Op, VT, Action); + } + void setOperationAction(ArrayRef<unsigned> Ops, ArrayRef<MVT> VTs, + LegalizeAction Action) { + for (auto VT : VTs) + setOperationAction(Ops, VT, Action); + } /// Indicate that the specified load with extension does not work with the /// specified type and indicate what to do about it. @@ -2244,6 +2324,16 @@ protected: LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &= ~((uint16_t)0xF << Shift); LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |= (uint16_t)Action << Shift; } + void setLoadExtAction(ArrayRef<unsigned> ExtTypes, MVT ValVT, MVT MemVT, + LegalizeAction Action) { + for (auto ExtType : ExtTypes) + setLoadExtAction(ExtType, ValVT, MemVT, Action); + } + void setLoadExtAction(ArrayRef<unsigned> ExtTypes, MVT ValVT, + ArrayRef<MVT> MemVTs, LegalizeAction Action) { + for (auto MemVT : MemVTs) + setLoadExtAction(ExtTypes, ValVT, MemVT, Action); + } /// Indicate that the specified truncating store does not work with the /// specified type and indicate what to do about it. @@ -2257,8 +2347,16 @@ protected: /// /// NOTE: All indexed mode loads are initialized to Expand in /// TargetLowering.cpp - void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action) { - setIndexedModeAction(IdxMode, VT, IMAB_Load, Action); + void setIndexedLoadAction(ArrayRef<unsigned> IdxModes, MVT VT, + LegalizeAction Action) { + for (auto IdxMode : IdxModes) + setIndexedModeAction(IdxMode, VT, IMAB_Load, Action); + } + + void setIndexedLoadAction(ArrayRef<unsigned> IdxModes, ArrayRef<MVT> VTs, + LegalizeAction Action) { + for (auto VT : VTs) + setIndexedLoadAction(IdxModes, VT, Action); } /// Indicate that the specified indexed store does or does not work with the @@ -2266,8 +2364,16 @@ protected: /// /// NOTE: All indexed mode stores are initialized to Expand in /// TargetLowering.cpp - void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action) { - setIndexedModeAction(IdxMode, VT, IMAB_Store, Action); + void setIndexedStoreAction(ArrayRef<unsigned> IdxModes, MVT VT, + LegalizeAction Action) { + for (auto IdxMode : IdxModes) + setIndexedModeAction(IdxMode, VT, IMAB_Store, Action); + } + + void setIndexedStoreAction(ArrayRef<unsigned> IdxModes, ArrayRef<MVT> VTs, + LegalizeAction Action) { + for (auto VT : VTs) + setIndexedStoreAction(IdxModes, VT, Action); } /// Indicate that the specified indexed masked load does or does not work with @@ -2292,17 +2398,24 @@ protected: /// Indicate that the specified condition code is or isn't supported on the /// target and indicate what to do about it. - void setCondCodeAction(ISD::CondCode CC, MVT VT, + void setCondCodeAction(ArrayRef<ISD::CondCode> CCs, MVT VT, LegalizeAction Action) { - assert(VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) && - "Table isn't big enough!"); - assert((unsigned)Action < 0x10 && "too many bits for bitfield array"); - /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the 32-bit - /// value and the upper 29 bits index into the second dimension of the array - /// to select what 32-bit value to use. - uint32_t Shift = 4 * (VT.SimpleTy & 0x7); - CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift); - CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift; + for (auto CC : CCs) { + assert(VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) && + "Table isn't big enough!"); + assert((unsigned)Action < 0x10 && "too many bits for bitfield array"); + /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the + /// 32-bit value and the upper 29 bits index into the second dimension of + /// the array to select what 32-bit value to use. + uint32_t Shift = 4 * (VT.SimpleTy & 0x7); + CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift); + CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift; + } + } + void setCondCodeAction(ArrayRef<ISD::CondCode> CCs, ArrayRef<MVT> VTs, + LegalizeAction Action) { + for (auto VT : VTs) + setCondCodeAction(CCs, VT, Action); } /// If Opc/OrigVT is specified as being promoted, the promotion code defaults @@ -2323,9 +2436,11 @@ protected: /// Targets should invoke this method for each target independent node that /// they want to provide a custom DAG combiner for by implementing the /// PerformDAGCombine virtual method. - void setTargetDAGCombine(ISD::NodeType NT) { - assert(unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray)); - TargetDAGCombineArray[NT >> 3] |= 1 << (NT&7); + void setTargetDAGCombine(ArrayRef<ISD::NodeType> NTs) { + for (auto NT : NTs) { + assert(unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray)); + TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7); + } } /// Set the target's minimum function alignment. @@ -2510,6 +2625,10 @@ public: case ISD::FMAXNUM_IEEE: case ISD::FMINIMUM: case ISD::FMAXIMUM: + case ISD::AVGFLOORS: + case ISD::AVGFLOORU: + case ISD::AVGCEILS: + case ISD::AVGCEILU: return true; default: return false; } @@ -2653,6 +2772,10 @@ public: return false; } + /// Return true if this constant should be sign extended when promoting to + /// a larger type. + virtual bool signExtendConstant(const ConstantInt *C) const { return false; } + /// Return true if sinking I's operands to the same basic block as I is /// profitable, e.g. because the operands can be folded into a target /// instruction during instruction selection. After calling the function @@ -2851,6 +2974,14 @@ public: return false; } + /// Return true if pulling a binary operation into a select with an identity + /// constant is profitable. This is the inverse of an IR transform. + /// Example: X + (Cond ? Y : 0) --> Cond ? (X + Y) : X + virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, + EVT VT) const { + return false; + } + /// Return true if it is beneficial to convert a load of a constant to /// just the constant itself. /// On some targets it might be more efficient to use a combination of @@ -2940,6 +3071,10 @@ public: void setLibcallName(RTLIB::Libcall Call, const char *Name) { LibcallRoutineNames[Call] = Name; } + void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) { + for (auto Call : Calls) + setLibcallName(Call, Name); + } /// Get the libcall routine name for the specified libcall. const char *getLibcallName(RTLIB::Libcall Call) const { @@ -3421,11 +3556,13 @@ public: /// Determines the optimal series of memory ops to replace the memset / memcpy. /// Return true if the number of memory ops is below the threshold (Limit). + /// Note that this is always the case when Limit is ~0. /// It returns the types of the sequence of memory ops to perform /// memset / memcpy by reference. - bool findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit, - const MemOp &Op, unsigned DstAS, unsigned SrcAS, - const AttributeList &FuncAttributes) const; + virtual bool + findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit, + const MemOp &Op, unsigned DstAS, unsigned SrcAS, + const AttributeList &FuncAttributes) const; /// Check to see if the specified operand of the specified instruction is a /// constant integer. If so, check to see if there are any bits set in the @@ -3534,9 +3671,16 @@ public: /// Helper wrapper around SimplifyDemandedVectorElts. /// Adds Op back to the worklist upon success. bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, - APInt &KnownUndef, APInt &KnownZero, DAGCombinerInfo &DCI) const; + /// Return true if the target supports simplifying demanded vector elements by + /// converting them to undefs. + virtual bool + shouldSimplifyDemandedVectorElts(SDValue Op, + const TargetLoweringOpt &TLO) const { + return true; + } + /// Determine which of the bits specified in Mask are known to be either zero /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts /// argument allows us to only collect the known bits that are shared by the @@ -3653,6 +3797,12 @@ public: APInt &UndefElts, unsigned Depth = 0) const; + /// Returns true if the given Opc is considered a canonical constant for the + /// target, which should not be transformed back into a BUILD_VECTOR. + virtual bool isTargetCanonicalConstantNode(SDValue Op) const { + return Op.getOpcode() == ISD::SPLAT_VECTOR; + } + struct DAGCombinerInfo { void *DC; // The DAG Combiner object. CombineLevel Level; @@ -3805,7 +3955,7 @@ public: if (Neg && Cost == NegatibleCost::Cheaper) return Neg; // Remove the new created node to avoid the side effect to the DAG. - if (Neg && Neg.getNode()->use_empty()) + if (Neg && Neg->use_empty()) DAG.RemoveDeadNode(Neg.getNode()); return SDValue(); } @@ -4270,6 +4420,7 @@ public: C_Register, // Constraint represents specific register(s). C_RegisterClass, // Constraint represents any of register(s) in class. C_Memory, // Memory constraint. + C_Address, // Address constraint. C_Immediate, // Requires an immediate. C_Other, // Something else. C_Unknown // Unsupported constraint. @@ -4374,6 +4525,8 @@ public: return InlineAsm::Constraint_o; if (ConstraintCode == "X") return InlineAsm::Constraint_X; + if (ConstraintCode == "p") + return InlineAsm::Constraint_p; return InlineAsm::Constraint_Unknown; } @@ -4410,6 +4563,14 @@ public: SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) const; + /// Targets may override this function to provide custom SREM lowering for + /// power-of-2 denominators. If the target returns an empty SDValue, LLVM + /// assumes SREM is expensive and replaces it with a series of other integer + /// operations. + virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, + SelectionDAG &DAG, + SmallVectorImpl<SDNode *> &Created) const; + /// Indicate whether this target prefers to combine FDIVs with the same /// divisor. If the transform should never be done, return zero. If the /// transform should be done, return the minimum number of divisor uses @@ -4442,6 +4603,13 @@ public: return SDValue(); } + /// Try to convert the fminnum/fmaxnum to a compare/select sequence. This is + /// required for correctness since InstCombine might have canonicalized a + /// fcmp+select sequence to a FMINNUM/FMAXNUM intrinsic. If we were to fall + /// through to the default expansion/soften to libcall, we might introduce a + /// link-time dependency on libm into a file that originally did not have one. + SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const; + /// Return a reciprocal estimate value for the input operand. /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or /// 'Enabled' as set by a potential default override attribute. @@ -4554,6 +4722,16 @@ public: /// \returns The expansion result SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const; + /// Expand check for floating point class. + /// \param ResultVT The type of intrinsic call result. + /// \param Op The tested value. + /// \param Test The test to perform. + /// \param Flags The optimization flags. + /// \returns The expansion result or SDValue() if it fails. + SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, unsigned Test, + SDNodeFlags Flags, const SDLoc &DL, + SelectionDAG &DAG) const; + /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes, /// vector nodes can only succeed if all operations are legal/custom. /// \param N Node to expand @@ -4693,28 +4871,32 @@ public: /// method accepts vectors as its arguments. SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const; - /// Legalize a SETCC with given LHS and RHS and condition code CC on the - /// current target. + /// Legalize a SETCC or VP_SETCC with given LHS and RHS and condition code CC + /// on the current target. A VP_SETCC will additionally be given a Mask + /// and/or EVL not equal to SDValue(). /// /// If the SETCC has been legalized using AND / OR, then the legalized node /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert - /// will be set to false. + /// will be set to false. This will also hold if the VP_SETCC has been + /// legalized using VP_AND / VP_OR. /// - /// If the SETCC has been legalized by using getSetCCSwappedOperands(), - /// then the values of LHS and RHS will be swapped, CC will be set to the - /// new condition, and NeedInvert will be set to false. + /// If the SETCC / VP_SETCC has been legalized by using + /// getSetCCSwappedOperands(), then the values of LHS and RHS will be + /// swapped, CC will be set to the new condition, and NeedInvert will be set + /// to false. /// - /// If the SETCC has been legalized using the inverse condcode, then LHS and - /// RHS will be unchanged, CC will set to the inverted condcode, and - /// NeedInvert will be set to true. The caller must invert the result of the - /// SETCC with SelectionDAG::getLogicalNOT() or take equivalent action to swap - /// the effect of a true/false result. + /// If the SETCC / VP_SETCC has been legalized using the inverse condcode, + /// then LHS and RHS will be unchanged, CC will set to the inverted condcode, + /// and NeedInvert will be set to true. The caller must invert the result of + /// the SETCC with SelectionDAG::getLogicalNOT() or take equivalent action to + /// swap the effect of a true/false result. /// - /// \returns true if the SetCC has been legalized, false if it hasn't. + /// \returns true if the SETCC / VP_SETCC has been legalized, false if it + /// hasn't. bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, - SDValue &RHS, SDValue &CC, bool &NeedInvert, - const SDLoc &dl, SDValue &Chain, - bool IsSignaling = false) const; + SDValue &RHS, SDValue &CC, SDValue Mask, + SDValue EVL, bool &NeedInvert, const SDLoc &dl, + SDValue &Chain, bool IsSignaling = false) const; //===--------------------------------------------------------------------===// // Instruction Emitting Hooks @@ -4766,10 +4948,6 @@ public: // combiner can fold the new nodes. SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const; - /// Give targets the chance to reduce the number of distinct addresing modes. - ISD::MemIndexType getCanonicalIndexType(ISD::MemIndexType IndexType, - EVT MemVT, SDValue Offsets) const; - private: SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &DL, DAGCombinerInfo &DCI) const; diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 2c8b17807f7c..08267d70906a 100644 --- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -16,6 +16,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/BinaryFormat/XCOFF.h" +#include "llvm/MC/MCExpr.h" #include "llvm/Target/TargetLoweringObjectFile.h" namespace llvm { @@ -118,6 +119,9 @@ public: void Initialize(MCContext &Ctx, const TargetMachine &TM) override; + MCSection *getStaticDtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + /// Emit the module flags that specify the garbage collection information. void emitModuleMetadata(MCStreamer &Streamer, Module &M) const override; @@ -282,6 +286,13 @@ public: MCSymbol *getFunctionEntryPointSymbol(const GlobalValue *Func, const TargetMachine &TM) const override; + + /// For functions, this will return the LSDA section. If option + /// -ffunction-sections is on, this will return a unique csect with the + /// function name appended to .gcc_except_table as a suffix of the LSDA + /// section name. + MCSection *getSectionForLSDA(const Function &F, const MCSymbol &FnSym, + const TargetMachine &TM) const override; }; class TargetLoweringObjectFileGOFF : public TargetLoweringObjectFile { diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h index 9b13b61fc9de..8d7086d02c8a 100644 --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -345,6 +345,9 @@ protected: // Helper to verify the analysis is really immutable. void setOpt(bool &Opt, bool Val); + /// Return true if register allocator is specified by -regalloc=override. + bool isCustomizedRegAlloc(); + /// Methods with trivial inline returns are convenient points in the common /// codegen pass pipeline where targets may insert passes. Methods with /// out-of-line standard implementations are major CodeGen stages called by diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index c3b842052ef5..04369a5bfe0d 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -29,7 +29,6 @@ #include "llvm/Support/Printable.h" #include <cassert> #include <cstdint> -#include <functional> namespace llvm { @@ -56,6 +55,8 @@ public: const LaneBitmask LaneMask; /// Classes with a higher priority value are assigned first by register /// allocators using a greedy heuristic. The value is in the range [0,63]. + /// Values >= 32 should be used with care since they may overlap with other + /// fields in the allocator's priority heuristics. const uint8_t AllocationPriority; /// Configurable target specific flags. const uint8_t TSFlags; @@ -415,19 +416,11 @@ public: /// Returns true if the two registers are equal or alias each other. /// The registers may be virtual registers. - bool regsOverlap(Register regA, Register regB) const { - if (regA == regB) return true; - if (!regA.isPhysical() || !regB.isPhysical()) - return false; - - // Regunits are numerically ordered. Find a common unit. - MCRegUnitIterator RUA(regA.asMCReg(), this); - MCRegUnitIterator RUB(regB.asMCReg(), this); - do { - if (*RUA == *RUB) return true; - if (*RUA < *RUB) ++RUA; - else ++RUB; - } while (RUA.isValid() && RUB.isValid()); + bool regsOverlap(Register RegA, Register RegB) const { + if (RegA == RegB) + return true; + if (RegA.isPhysical() && RegB.isPhysical()) + return MCRegisterInfo::regsOverlap(RegA.asMCReg(), RegB.asMCReg()); return false; } @@ -567,6 +560,24 @@ public: virtual bool isCalleeSavedPhysReg(MCRegister PhysReg, const MachineFunction &MF) const; + /// Returns true if PhysReg can be used as an argument to a function. + virtual bool isArgumentRegister(const MachineFunction &MF, + MCRegister PhysReg) const { + return false; + } + + /// Returns true if PhysReg is a fixed register. + virtual bool isFixedRegister(const MachineFunction &MF, + MCRegister PhysReg) const { + return false; + } + + /// Returns true if PhysReg is a general purpose register. + virtual bool isGeneralPurposeRegister(const MachineFunction &MF, + MCRegister PhysReg) const { + return false; + } + /// Prior to adding the live-out mask to a stackmap or patchpoint /// instruction, provide the target the opportunity to adjust it (mainly to /// remove pseudo-registers that should be ignored). @@ -1067,6 +1078,14 @@ public: return false; } + /// When prioritizing live ranges in register allocation, if this hook returns + /// true then the AllocationPriority of the register class will be treated as + /// more important than whether the range is local to a basic block or global. + virtual bool + regClassPriorityTrumpsGlobalness(const MachineFunction &MF) const { + return false; + } + //===--------------------------------------------------------------------===// /// Debug information queries. diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h index 3fac2f688dd8..dbd678b75d05 100644 --- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -13,12 +13,10 @@ #ifndef LLVM_CODEGEN_TARGETSUBTARGETINFO_H #define LLVM_CODEGEN_TARGETSUBTARGETINFO_H -#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/PBQPRAConstraint.h" -#include "llvm/CodeGen/ScheduleDAGMutation.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/Support/CodeGen.h" @@ -27,6 +25,9 @@ namespace llvm { +class APInt; +class MachineFunction; +class ScheduleDAGMutation; class CallLowering; class InlineAsmLowering; class InstrItineraryData; @@ -272,11 +273,6 @@ public: /// a finer grain to tune the register allocator. virtual bool enableRALocalReassignment(CodeGenOpt::Level OptLevel) const; - /// True if the subtarget should consider the cost of local intervals - /// created by a split candidate when choosing the best split candidate. This - /// heuristic may be compile time intensive. - virtual bool enableAdvancedRASplitCost() const; - /// Enable use of alias analysis during code generation (during MI /// scheduling, DAGCombine, etc.). virtual bool useAA() const; diff --git a/llvm/include/llvm/CodeGen/TileShapeInfo.h b/llvm/include/llvm/CodeGen/TileShapeInfo.h index 4e574bd96cca..1b5f902139fb 100644 --- a/llvm/include/llvm/CodeGen/TileShapeInfo.h +++ b/llvm/include/llvm/CodeGen/TileShapeInfo.h @@ -38,7 +38,7 @@ public: ShapeT() : Row(nullptr), Col(nullptr), RowImm(InvalidImmShape), ColImm(InvalidImmShape) {} - bool operator==(const ShapeT &Shape) { + bool operator==(const ShapeT &Shape) const { MachineOperand *R = Shape.Row; MachineOperand *C = Shape.Col; if (!R || !C) @@ -52,7 +52,7 @@ public: return false; } - bool operator!=(const ShapeT &Shape) { return !(*this == Shape); } + bool operator!=(const ShapeT &Shape) const { return !(*this == Shape); } MachineOperand *getRow() const { return Row; } diff --git a/llvm/include/llvm/CodeGen/ValueTypes.h b/llvm/include/llvm/CodeGen/ValueTypes.h index 7b17b98d5c55..48d265476ca8 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.h +++ b/llvm/include/llvm/CodeGen/ValueTypes.h @@ -19,7 +19,6 @@ #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/TypeSize.h" -#include "llvm/Support/WithColor.h" #include <cassert> #include <cstdint> #include <string> @@ -365,6 +364,12 @@ namespace llvm { return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()}; } + // Return the number of bytes overwritten by a store of this value type or + // this value type's element type in the case of a vector. + uint64_t getScalarStoreSize() const { + return getScalarType().getStoreSize().getFixedSize(); + } + /// Return the number of bits overwritten by a store of the specified value /// type. /// diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index 7f989e08e9bf..2194800b7ba9 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -20,204 +20,211 @@ class ValueType<int size, int value> { def OtherVT : ValueType<0, 1>; // "Other" value def i1 : ValueType<1, 2>; // One bit boolean value -def i8 : ValueType<8, 3>; // 8-bit integer value -def i16 : ValueType<16, 4>; // 16-bit integer value -def i32 : ValueType<32, 5>; // 32-bit integer value -def i64 : ValueType<64, 6>; // 64-bit integer value -def i128 : ValueType<128, 7>; // 128-bit integer value - -def bf16 : ValueType<16, 8>; // 16-bit brain floating point value -def f16 : ValueType<16, 9>; // 16-bit floating point value -def f32 : ValueType<32, 10>; // 32-bit floating point value -def f64 : ValueType<64, 11>; // 64-bit floating point value -def f80 : ValueType<80, 12>; // 80-bit floating point value -def f128 : ValueType<128, 13>; // 128-bit floating point value -def ppcf128 : ValueType<128, 14>; // PPC 128-bit floating point value - -def v1i1 : ValueType<1, 15>; // 1 x i1 vector value -def v2i1 : ValueType<2, 16>; // 2 x i1 vector value -def v4i1 : ValueType<4, 17>; // 4 x i1 vector value -def v8i1 : ValueType<8, 18>; // 8 x i1 vector value -def v16i1 : ValueType<16, 19>; // 16 x i1 vector value -def v32i1 : ValueType<32, 20>; // 32 x i1 vector value -def v64i1 : ValueType<64, 21>; // 64 x i1 vector value -def v128i1 : ValueType<128, 22>; // 128 x i1 vector value -def v256i1 : ValueType<256, 23>; // 256 x i1 vector value -def v512i1 : ValueType<512, 24>; // 512 x i1 vector value -def v1024i1 : ValueType<1024, 25>; // 1024 x i1 vector value - -def v1i8 : ValueType<8, 26>; // 1 x i8 vector value -def v2i8 : ValueType<16, 27>; // 2 x i8 vector value -def v4i8 : ValueType<32, 28>; // 4 x i8 vector value -def v8i8 : ValueType<64, 29>; // 8 x i8 vector value -def v16i8 : ValueType<128, 30>; // 16 x i8 vector value -def v32i8 : ValueType<256, 31>; // 32 x i8 vector value -def v64i8 : ValueType<512, 32>; // 64 x i8 vector value -def v128i8 : ValueType<1024, 33>; // 128 x i8 vector value -def v256i8 : ValueType<2048, 34>; // 256 x i8 vector value -def v512i8 : ValueType<4096, 35>; // 512 x i8 vector value -def v1024i8 : ValueType<8192, 36>; // 1024 x i8 vector value - -def v1i16 : ValueType<16, 37>; // 1 x i16 vector value -def v2i16 : ValueType<32, 38>; // 2 x i16 vector value -def v3i16 : ValueType<48, 39>; // 3 x i16 vector value -def v4i16 : ValueType<64, 40>; // 4 x i16 vector value -def v8i16 : ValueType<128, 41>; // 8 x i16 vector value -def v16i16 : ValueType<256, 42>; // 16 x i16 vector value -def v32i16 : ValueType<512, 43>; // 32 x i16 vector value -def v64i16 : ValueType<1024, 44>; // 64 x i16 vector value -def v128i16 : ValueType<2048, 45>; // 128 x i16 vector value -def v256i16 : ValueType<4096, 46>; // 256 x i16 vector value -def v512i16 : ValueType<8192, 47>; // 512 x i16 vector value - -def v1i32 : ValueType<32, 48>; // 1 x i32 vector value -def v2i32 : ValueType<64, 49>; // 2 x i32 vector value -def v3i32 : ValueType<96, 50>; // 3 x i32 vector value -def v4i32 : ValueType<128, 51>; // 4 x i32 vector value -def v5i32 : ValueType<160, 52>; // 5 x i32 vector value -def v6i32 : ValueType<192, 53>; // 6 x f32 vector value -def v7i32 : ValueType<224, 54>; // 7 x f32 vector value -def v8i32 : ValueType<256, 55>; // 8 x i32 vector value -def v16i32 : ValueType<512, 56>; // 16 x i32 vector value -def v32i32 : ValueType<1024, 57>; // 32 x i32 vector value -def v64i32 : ValueType<2048, 58>; // 64 x i32 vector value -def v128i32 : ValueType<4096, 59>; // 128 x i32 vector value -def v256i32 : ValueType<8192, 60>; // 256 x i32 vector value -def v512i32 : ValueType<16384, 61>; // 512 x i32 vector value -def v1024i32 : ValueType<32768, 62>; // 1024 x i32 vector value -def v2048i32 : ValueType<65536, 63>; // 2048 x i32 vector value - -def v1i64 : ValueType<64, 64>; // 1 x i64 vector value -def v2i64 : ValueType<128, 65>; // 2 x i64 vector value -def v3i64 : ValueType<192, 66>; // 3 x i64 vector value -def v4i64 : ValueType<256, 67>; // 4 x i64 vector value -def v8i64 : ValueType<512, 68>; // 8 x i64 vector value -def v16i64 : ValueType<1024, 69>; // 16 x i64 vector value -def v32i64 : ValueType<2048, 70>; // 32 x i64 vector value -def v64i64 : ValueType<4096, 71>; // 64 x i64 vector value -def v128i64 : ValueType<8192, 72>; // 128 x i64 vector value -def v256i64 : ValueType<16384, 73>; // 256 x i64 vector value - -def v1i128 : ValueType<128, 74>; // 1 x i128 vector value - -def v1f16 : ValueType<16, 75>; // 1 x f16 vector value -def v2f16 : ValueType<32, 76>; // 2 x f16 vector value -def v3f16 : ValueType<48, 77>; // 3 x f16 vector value -def v4f16 : ValueType<64, 78>; // 4 x f16 vector value -def v8f16 : ValueType<128, 79>; // 8 x f16 vector value -def v16f16 : ValueType<256, 80>; // 16 x f16 vector value -def v32f16 : ValueType<512, 81>; // 32 x f16 vector value -def v64f16 : ValueType<1024, 82>; // 64 x f16 vector value -def v128f16 : ValueType<2048, 83>; // 128 x f16 vector value -def v256f16 : ValueType<4096, 84>; // 256 x f16 vector value -def v512f16 : ValueType<8192, 85>; // 512 x f16 vector value - -def v2bf16 : ValueType<32, 86>; // 2 x bf16 vector value -def v3bf16 : ValueType<48, 87>; // 3 x bf16 vector value -def v4bf16 : ValueType<64, 88>; // 4 x bf16 vector value -def v8bf16 : ValueType<128, 89>; // 8 x bf16 vector value -def v16bf16 : ValueType<256, 90>; // 16 x bf16 vector value -def v32bf16 : ValueType<512, 91>; // 32 x bf16 vector value -def v64bf16 : ValueType<1024, 92>; // 64 x bf16 vector value -def v128bf16 : ValueType<2048, 93>; // 128 x bf16 vector value - -def v1f32 : ValueType<32, 94>; // 1 x f32 vector value -def v2f32 : ValueType<64, 95>; // 2 x f32 vector value -def v3f32 : ValueType<96, 96>; // 3 x f32 vector value -def v4f32 : ValueType<128, 97>; // 4 x f32 vector value -def v5f32 : ValueType<160, 98>; // 5 x f32 vector value -def v6f32 : ValueType<192, 99>; // 6 x f32 vector value -def v7f32 : ValueType<224, 100>; // 7 x f32 vector value -def v8f32 : ValueType<256, 101>; // 8 x f32 vector value -def v16f32 : ValueType<512, 102>; // 16 x f32 vector value -def v32f32 : ValueType<1024, 103>; // 32 x f32 vector value -def v64f32 : ValueType<2048, 104>; // 64 x f32 vector value -def v128f32 : ValueType<4096, 105>; // 128 x f32 vector value -def v256f32 : ValueType<8192, 106>; // 256 x f32 vector value -def v512f32 : ValueType<16384, 107>; // 512 x f32 vector value -def v1024f32 : ValueType<32768, 108>; // 1024 x f32 vector value -def v2048f32 : ValueType<65536, 109>; // 2048 x f32 vector value - -def v1f64 : ValueType<64, 110>; // 1 x f64 vector value -def v2f64 : ValueType<128, 111>; // 2 x f64 vector value -def v3f64 : ValueType<192, 112>; // 3 x f64 vector value -def v4f64 : ValueType<256, 113>; // 4 x f64 vector value -def v8f64 : ValueType<512, 114>; // 8 x f64 vector value -def v16f64 : ValueType<1024, 115>; // 16 x f64 vector value -def v32f64 : ValueType<2048, 116>; // 32 x f64 vector value -def v64f64 : ValueType<4096, 117>; // 64 x f64 vector value -def v128f64 : ValueType<8192, 118>; // 128 x f64 vector value -def v256f64 : ValueType<16384, 119>; // 256 x f64 vector value - -def nxv1i1 : ValueType<1, 120>; // n x 1 x i1 vector value -def nxv2i1 : ValueType<2, 121>; // n x 2 x i1 vector value -def nxv4i1 : ValueType<4, 122>; // n x 4 x i1 vector value -def nxv8i1 : ValueType<8, 123>; // n x 8 x i1 vector value -def nxv16i1 : ValueType<16, 124>; // n x 16 x i1 vector value -def nxv32i1 : ValueType<32, 125>; // n x 32 x i1 vector value -def nxv64i1 : ValueType<64, 126>; // n x 64 x i1 vector value - -def nxv1i8 : ValueType<8, 127>; // n x 1 x i8 vector value -def nxv2i8 : ValueType<16, 128>; // n x 2 x i8 vector value -def nxv4i8 : ValueType<32, 129>; // n x 4 x i8 vector value -def nxv8i8 : ValueType<64, 130>; // n x 8 x i8 vector value -def nxv16i8 : ValueType<128, 131>; // n x 16 x i8 vector value -def nxv32i8 : ValueType<256, 132>; // n x 32 x i8 vector value -def nxv64i8 : ValueType<512, 133>; // n x 64 x i8 vector value - -def nxv1i16 : ValueType<16, 134>; // n x 1 x i16 vector value -def nxv2i16 : ValueType<32, 135>; // n x 2 x i16 vector value -def nxv4i16 : ValueType<64, 136>; // n x 4 x i16 vector value -def nxv8i16 : ValueType<128, 137>; // n x 8 x i16 vector value -def nxv16i16 : ValueType<256, 138>; // n x 16 x i16 vector value -def nxv32i16 : ValueType<512, 139>; // n x 32 x i16 vector value - -def nxv1i32 : ValueType<32, 140>; // n x 1 x i32 vector value -def nxv2i32 : ValueType<64, 141>; // n x 2 x i32 vector value -def nxv4i32 : ValueType<128, 142>; // n x 4 x i32 vector value -def nxv8i32 : ValueType<256, 143>; // n x 8 x i32 vector value -def nxv16i32 : ValueType<512, 144>; // n x 16 x i32 vector value -def nxv32i32 : ValueType<1024, 145>; // n x 32 x i32 vector value - -def nxv1i64 : ValueType<64, 146>; // n x 1 x i64 vector value -def nxv2i64 : ValueType<128, 147>; // n x 2 x i64 vector value -def nxv4i64 : ValueType<256, 148>; // n x 4 x i64 vector value -def nxv8i64 : ValueType<512, 149>; // n x 8 x i64 vector value -def nxv16i64 : ValueType<1024, 150>; // n x 16 x i64 vector value -def nxv32i64 : ValueType<2048, 151>; // n x 32 x i64 vector value - -def nxv1f16 : ValueType<16, 152>; // n x 1 x f16 vector value -def nxv2f16 : ValueType<32, 153>; // n x 2 x f16 vector value -def nxv4f16 : ValueType<64, 154>; // n x 4 x f16 vector value -def nxv8f16 : ValueType<128, 155>; // n x 8 x f16 vector value -def nxv16f16 : ValueType<256, 156>; // n x 16 x f16 vector value -def nxv32f16 : ValueType<512, 157>; // n x 32 x f16 vector value - -def nxv1bf16 : ValueType<16, 158>; // n x 1 x bf16 vector value -def nxv2bf16 : ValueType<32, 159>; // n x 2 x bf16 vector value -def nxv4bf16 : ValueType<64, 160>; // n x 4 x bf16 vector value -def nxv8bf16 : ValueType<128, 161>; // n x 8 x bf16 vector value - -def nxv1f32 : ValueType<32, 162>; // n x 1 x f32 vector value -def nxv2f32 : ValueType<64, 163>; // n x 2 x f32 vector value -def nxv4f32 : ValueType<128, 164>; // n x 4 x f32 vector value -def nxv8f32 : ValueType<256, 165>; // n x 8 x f32 vector value -def nxv16f32 : ValueType<512, 166>; // n x 16 x f32 vector value - -def nxv1f64 : ValueType<64, 167>; // n x 1 x f64 vector value -def nxv2f64 : ValueType<128, 168>; // n x 2 x f64 vector value -def nxv4f64 : ValueType<256, 169>; // n x 4 x f64 vector value -def nxv8f64 : ValueType<512, 170>; // n x 8 x f64 vector value - -def x86mmx : ValueType<64, 171>; // X86 MMX value -def FlagVT : ValueType<0, 172>; // Pre-RA sched glue -def isVoid : ValueType<0, 173>; // Produces no value -def untyped : ValueType<8, 174>; // Produces an untyped value -def funcref : ValueType<0, 175>; // WebAssembly's funcref type -def externref : ValueType<0, 176>; // WebAssembly's externref type -def x86amx : ValueType<8192, 177>; // X86 AMX value -def i64x8 : ValueType<512, 178>; // 8 Consecutive GPRs (AArch64) - +def i2 : ValueType<2, 3>; // 2-bit integer value +def i4 : ValueType<4, 4>; // 4-bit integer value +def i8 : ValueType<8, 5>; // 8-bit integer value +def i16 : ValueType<16, 6>; // 16-bit integer value +def i32 : ValueType<32, 7>; // 32-bit integer value +def i64 : ValueType<64, 8>; // 64-bit integer value +def i128 : ValueType<128, 9>; // 128-bit integer value + +def bf16 : ValueType<16, 10>; // 16-bit brain floating point value +def f16 : ValueType<16, 11>; // 16-bit floating point value +def f32 : ValueType<32, 12>; // 32-bit floating point value +def f64 : ValueType<64, 13>; // 64-bit floating point value +def f80 : ValueType<80, 14>; // 80-bit floating point value +def f128 : ValueType<128, 15>; // 128-bit floating point value +def ppcf128 : ValueType<128, 16>; // PPC 128-bit floating point value + +def v1i1 : ValueType<1, 17>; // 1 x i1 vector value +def v2i1 : ValueType<2, 18>; // 2 x i1 vector value +def v4i1 : ValueType<4, 19>; // 4 x i1 vector value +def v8i1 : ValueType<8, 20>; // 8 x i1 vector value +def v16i1 : ValueType<16, 21>; // 16 x i1 vector value +def v32i1 : ValueType<32, 22>; // 32 x i1 vector value +def v64i1 : ValueType<64, 23>; // 64 x i1 vector value +def v128i1 : ValueType<128, 24>; // 128 x i1 vector value +def v256i1 : ValueType<256, 25>; // 256 x i1 vector value +def v512i1 : ValueType<512, 26>; // 512 x i1 vector value +def v1024i1 : ValueType<1024, 27>; // 1024 x i1 vector value + +def v128i2 : ValueType<256, 28>; // 128 x i2 vector value + +def v64i4 : ValueType<256, 29>; // 64 x i4 vector value + +def v1i8 : ValueType<8, 30>; // 1 x i8 vector value +def v2i8 : ValueType<16, 31>; // 2 x i8 vector value +def v4i8 : ValueType<32, 32>; // 4 x i8 vector value +def v8i8 : ValueType<64, 33>; // 8 x i8 vector value +def v16i8 : ValueType<128, 34>; // 16 x i8 vector value +def v32i8 : ValueType<256, 35>; // 32 x i8 vector value +def v64i8 : ValueType<512, 36>; // 64 x i8 vector value +def v128i8 : ValueType<1024, 37>; // 128 x i8 vector value +def v256i8 : ValueType<2048, 38>; // 256 x i8 vector value +def v512i8 : ValueType<4096, 39>; // 512 x i8 vector value +def v1024i8 : ValueType<8192, 40>; // 1024 x i8 vector value + +def v1i16 : ValueType<16, 41>; // 1 x i16 vector value +def v2i16 : ValueType<32, 42>; // 2 x i16 vector value +def v3i16 : ValueType<48, 43>; // 3 x i16 vector value +def v4i16 : ValueType<64, 44>; // 4 x i16 vector value +def v8i16 : ValueType<128, 45>; // 8 x i16 vector value +def v16i16 : ValueType<256, 46>; // 16 x i16 vector value +def v32i16 : ValueType<512, 47>; // 32 x i16 vector value +def v64i16 : ValueType<1024, 48>; // 64 x i16 vector value +def v128i16 : ValueType<2048, 49>; // 128 x i16 vector value +def v256i16 : ValueType<4096, 50>; // 256 x i16 vector value +def v512i16 : ValueType<8192, 51>; // 512 x i16 vector value + +def v1i32 : ValueType<32, 52>; // 1 x i32 vector value +def v2i32 : ValueType<64, 53>; // 2 x i32 vector value +def v3i32 : ValueType<96, 54>; // 3 x i32 vector value +def v4i32 : ValueType<128, 55>; // 4 x i32 vector value +def v5i32 : ValueType<160, 56>; // 5 x i32 vector value +def v6i32 : ValueType<192, 57>; // 6 x f32 vector value +def v7i32 : ValueType<224, 58>; // 7 x f32 vector value +def v8i32 : ValueType<256, 59>; // 8 x i32 vector value +def v16i32 : ValueType<512, 60>; // 16 x i32 vector value +def v32i32 : ValueType<1024, 61>; // 32 x i32 vector value +def v64i32 : ValueType<2048, 62>; // 64 x i32 vector value +def v128i32 : ValueType<4096, 63>; // 128 x i32 vector value +def v256i32 : ValueType<8192, 64>; // 256 x i32 vector value +def v512i32 : ValueType<16384, 65>; // 512 x i32 vector value +def v1024i32 : ValueType<32768, 66>; // 1024 x i32 vector value +def v2048i32 : ValueType<65536, 67>; // 2048 x i32 vector value + +def v1i64 : ValueType<64, 68>; // 1 x i64 vector value +def v2i64 : ValueType<128, 69>; // 2 x i64 vector value +def v3i64 : ValueType<192, 70>; // 3 x i64 vector value +def v4i64 : ValueType<256, 71>; // 4 x i64 vector value +def v8i64 : ValueType<512, 72>; // 8 x i64 vector value +def v16i64 : ValueType<1024, 73>; // 16 x i64 vector value +def v32i64 : ValueType<2048, 74>; // 32 x i64 vector value +def v64i64 : ValueType<4096, 75>; // 64 x i64 vector value +def v128i64 : ValueType<8192, 76>; // 128 x i64 vector value +def v256i64 : ValueType<16384, 77>; // 256 x i64 vector value + +def v1i128 : ValueType<128, 78>; // 1 x i128 vector value + +def v1f16 : ValueType<16, 79>; // 1 x f16 vector value +def v2f16 : ValueType<32, 80>; // 2 x f16 vector value +def v3f16 : ValueType<48, 81>; // 3 x f16 vector value +def v4f16 : ValueType<64, 82>; // 4 x f16 vector value +def v8f16 : ValueType<128, 83>; // 8 x f16 vector value +def v16f16 : ValueType<256, 84>; // 16 x f16 vector value +def v32f16 : ValueType<512, 85>; // 32 x f16 vector value +def v64f16 : ValueType<1024, 86>; // 64 x f16 vector value +def v128f16 : ValueType<2048, 87>; // 128 x f16 vector value +def v256f16 : ValueType<4096, 88>; // 256 x f16 vector value +def v512f16 : ValueType<8192, 89>; // 512 x f16 vector value + +def v2bf16 : ValueType<32, 90>; // 2 x bf16 vector value +def v3bf16 : ValueType<48, 91>; // 3 x bf16 vector value +def v4bf16 : ValueType<64, 92>; // 4 x bf16 vector value +def v8bf16 : ValueType<128, 93>; // 8 x bf16 vector value +def v16bf16 : ValueType<256, 94>; // 16 x bf16 vector value +def v32bf16 : ValueType<512, 95>; // 32 x bf16 vector value +def v64bf16 : ValueType<1024, 96>; // 64 x bf16 vector value +def v128bf16 : ValueType<2048, 97>; // 128 x bf16 vector value + +def v1f32 : ValueType<32, 98>; // 1 x f32 vector value +def v2f32 : ValueType<64, 99>; // 2 x f32 vector value +def v3f32 : ValueType<96, 100>; // 3 x f32 vector value +def v4f32 : ValueType<128, 101>; // 4 x f32 vector value +def v5f32 : ValueType<160, 102>; // 5 x f32 vector value +def v6f32 : ValueType<192, 103>; // 6 x f32 vector value +def v7f32 : ValueType<224, 104>; // 7 x f32 vector value +def v8f32 : ValueType<256, 105>; // 8 x f32 vector value +def v16f32 : ValueType<512, 106>; // 16 x f32 vector value +def v32f32 : ValueType<1024, 107>; // 32 x f32 vector value +def v64f32 : ValueType<2048, 108>; // 64 x f32 vector value +def v128f32 : ValueType<4096, 109>; // 128 x f32 vector value +def v256f32 : ValueType<8192, 110>; // 256 x f32 vector value +def v512f32 : ValueType<16384, 111>; // 512 x f32 vector value +def v1024f32 : ValueType<32768, 112>; // 1024 x f32 vector value +def v2048f32 : ValueType<65536, 113>; // 2048 x f32 vector value + +def v1f64 : ValueType<64, 114>; // 1 x f64 vector value +def v2f64 : ValueType<128, 115>; // 2 x f64 vector value +def v3f64 : ValueType<192, 116>; // 3 x f64 vector value +def v4f64 : ValueType<256, 117>; // 4 x f64 vector value +def v8f64 : ValueType<512, 118>; // 8 x f64 vector value +def v16f64 : ValueType<1024, 119>; // 16 x f64 vector value +def v32f64 : ValueType<2048, 120>; // 32 x f64 vector value +def v64f64 : ValueType<4096, 121>; // 64 x f64 vector value +def v128f64 : ValueType<8192, 122>; // 128 x f64 vector value +def v256f64 : ValueType<16384, 123>; // 256 x f64 vector value + +def nxv1i1 : ValueType<1, 124>; // n x 1 x i1 vector value +def nxv2i1 : ValueType<2, 125>; // n x 2 x i1 vector value +def nxv4i1 : ValueType<4, 126>; // n x 4 x i1 vector value +def nxv8i1 : ValueType<8, 127>; // n x 8 x i1 vector value +def nxv16i1 : ValueType<16, 128>; // n x 16 x i1 vector value +def nxv32i1 : ValueType<32, 129>; // n x 32 x i1 vector value +def nxv64i1 : ValueType<64, 130>; // n x 64 x i1 vector value + +def nxv1i8 : ValueType<8, 131>; // n x 1 x i8 vector value +def nxv2i8 : ValueType<16, 132>; // n x 2 x i8 vector value +def nxv4i8 : ValueType<32, 133>; // n x 4 x i8 vector value +def nxv8i8 : ValueType<64, 134>; // n x 8 x i8 vector value +def nxv16i8 : ValueType<128, 135>; // n x 16 x i8 vector value +def nxv32i8 : ValueType<256, 136>; // n x 32 x i8 vector value +def nxv64i8 : ValueType<512, 137>; // n x 64 x i8 vector value + +def nxv1i16 : ValueType<16, 138>; // n x 1 x i16 vector value +def nxv2i16 : ValueType<32, 139>; // n x 2 x i16 vector value +def nxv4i16 : ValueType<64, 140>; // n x 4 x i16 vector value +def nxv8i16 : ValueType<128, 141>; // n x 8 x i16 vector value +def nxv16i16 : ValueType<256, 142>; // n x 16 x i16 vector value +def nxv32i16 : ValueType<512, 143>; // n x 32 x i16 vector value + +def nxv1i32 : ValueType<32, 144>; // n x 1 x i32 vector value +def nxv2i32 : ValueType<64, 145>; // n x 2 x i32 vector value +def nxv4i32 : ValueType<128, 146>; // n x 4 x i32 vector value +def nxv8i32 : ValueType<256, 147>; // n x 8 x i32 vector value +def nxv16i32 : ValueType<512, 148>; // n x 16 x i32 vector value +def nxv32i32 : ValueType<1024, 149>; // n x 32 x i32 vector value + +def nxv1i64 : ValueType<64, 150>; // n x 1 x i64 vector value +def nxv2i64 : ValueType<128, 151>; // n x 2 x i64 vector value +def nxv4i64 : ValueType<256, 152>; // n x 4 x i64 vector value +def nxv8i64 : ValueType<512, 153>; // n x 8 x i64 vector value +def nxv16i64 : ValueType<1024, 154>; // n x 16 x i64 vector value +def nxv32i64 : ValueType<2048, 155>; // n x 32 x i64 vector value + +def nxv1f16 : ValueType<16, 156>; // n x 1 x f16 vector value +def nxv2f16 : ValueType<32, 157>; // n x 2 x f16 vector value +def nxv4f16 : ValueType<64, 158>; // n x 4 x f16 vector value +def nxv8f16 : ValueType<128, 159>; // n x 8 x f16 vector value +def nxv16f16 : ValueType<256, 160>; // n x 16 x f16 vector value +def nxv32f16 : ValueType<512, 161>; // n x 32 x f16 vector value + +def nxv1bf16 : ValueType<16, 162>; // n x 1 x bf16 vector value +def nxv2bf16 : ValueType<32, 163>; // n x 2 x bf16 vector value +def nxv4bf16 : ValueType<64, 164>; // n x 4 x bf16 vector value +def nxv8bf16 : ValueType<128, 165>; // n x 8 x bf16 vector value +def nxv16bf16 : ValueType<256, 166>; // n x 16 x bf16 vector value +def nxv32bf16 : ValueType<512, 167>; // n x 32 x bf16 vector value + +def nxv1f32 : ValueType<32, 168>; // n x 1 x f32 vector value +def nxv2f32 : ValueType<64, 169>; // n x 2 x f32 vector value +def nxv4f32 : ValueType<128, 170>; // n x 4 x f32 vector value +def nxv8f32 : ValueType<256, 171>; // n x 8 x f32 vector value +def nxv16f32 : ValueType<512, 172>; // n x 16 x f32 vector value + +def nxv1f64 : ValueType<64, 173>; // n x 1 x f64 vector value +def nxv2f64 : ValueType<128, 174>; // n x 2 x f64 vector value +def nxv4f64 : ValueType<256, 175>; // n x 4 x f64 vector value +def nxv8f64 : ValueType<512, 176>; // n x 8 x f64 vector value + +def x86mmx : ValueType<64, 177>; // X86 MMX value +def FlagVT : ValueType<0, 178>; // Pre-RA sched glue +def isVoid : ValueType<0, 179>; // Produces no value +def untyped : ValueType<8, 180>; // Produces an untyped value +def funcref : ValueType<0, 181>; // WebAssembly's funcref type +def externref : ValueType<0, 182>; // WebAssembly's externref type +def x86amx : ValueType<8192, 183>; // X86 AMX value +def i64x8 : ValueType<512, 184>; // 8 Consecutive GPRs (AArch64) def token : ValueType<0, 248>; // TokenTy def MetadataVT : ValueType<0, 249>; // Metadata diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h index 4f1c666df35f..0b2e033bd97a 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h @@ -11,18 +11,26 @@ #include "llvm/CodeGen/AccelTable.h" #include "llvm/CodeGen/NonRelocatableStringpool.h" -#include "llvm/DWARFLinker/DWARFLinkerDeclContext.h" -#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/MC/MCDwarf.h" +#include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARF/DWARFDie.h" #include <map> namespace llvm { +class DWARFContext; +class DWARFExpression; +class DWARFUnit; +class DataExtractor; +class DeclContextTree; +struct MCDwarfLineTableParams; +template <typename T> class SmallVectorImpl; enum class DwarfLinkerClient { Dsymutil, LLD, General }; /// The kind of accelerator tables we should emit. -enum class AccelTableKind { +enum class DwarfLinkerAccelTableKind : uint8_t { + None, Apple, ///< .apple_names, .apple_namespaces, .apple_types, .apple_objc. Dwarf, ///< DWARF v5 .debug_names. Default, ///< Dwarf for DWARF5 or later, Apple otherwise. @@ -56,28 +64,21 @@ class AddressesMap { public: virtual ~AddressesMap(); - /// Returns true if represented addresses are from linked file. - /// Returns false if represented addresses are from not-linked - /// object file. - virtual bool areRelocationsResolved() const = 0; - /// Checks that there are valid relocations against a .debug_info /// section. virtual bool hasValidRelocs() = 0; - /// Checks that the specified DIE has a DW_AT_Location attribute - /// that references into a live code section. - /// + /// Checks that the specified variable \p DIE references live code section. + /// Allowed kind of input die: DW_TAG_variable, DW_TAG_constant. /// \returns true and sets Info.InDebugMap if it is the case. - virtual bool hasLiveMemoryLocation(const DWARFDie &DIE, - CompileUnit::DIEInfo &Info) = 0; + virtual bool isLiveVariable(const DWARFDie &DIE, + CompileUnit::DIEInfo &Info) = 0; - /// Checks that the specified DIE has a DW_AT_Low_pc attribute - /// that references into a live code section. - /// + /// Checks that the specified subprogram \p DIE references live code section. + /// Allowed kind of input die: DW_TAG_subprogram, DW_TAG_label. /// \returns true and sets Info.InDebugMap if it is the case. - virtual bool hasLiveAddressRange(const DWARFDie &DIE, - CompileUnit::DIEInfo &Info) = 0; + virtual bool isLiveSubprogram(const DWARFDie &DIE, + CompileUnit::DIEInfo &Info) = 0; /// Apply the valid relocations to the buffer \p Data, taking into /// account that Data is at \p BaseOffset in the .debug_info section. @@ -272,6 +273,9 @@ public: /// Print statistics to standard output. void setStatistics(bool Statistics) { Options.Statistics = Statistics; } + /// Verify the input DWARF. + void setVerifyInputDWARF(bool Verify) { Options.VerifyInputDWARF = Verify; } + /// Do not emit linked dwarf info. void setNoOutput(bool NoOut) { Options.NoOutput = NoOut; } @@ -290,7 +294,7 @@ public: void setNumThreads(unsigned NumThreads) { Options.Threads = NumThreads; } /// Set kind of accelerator tables to be generated. - void setAccelTableKind(AccelTableKind Kind) { + void setAccelTableKind(DwarfLinkerAccelTableKind Kind) { Options.TheAccelTableKind = Kind; } @@ -361,6 +365,8 @@ private: /// Given a DIE, update its incompleteness based on whether the DIEs it /// references are incomplete. UpdateRefIncompleteness, + /// Given a DIE, mark it as ODR Canonical if applicable. + MarkODRCanonicalDie, }; /// This class represents an item in the work list. The type defines what kind @@ -389,6 +395,9 @@ private: AncestorIdx(AncestorIdx) {} }; + /// Verify the given DWARF file. + bool verify(const DWARFFile &File); + /// returns true if we need to translate strings. bool needToTranslateStrings() { return StringsTranslator != nullptr; } @@ -457,6 +466,10 @@ private: const DWARFFile &File, SmallVectorImpl<WorklistItem> &Worklist); + /// Mark context corresponding to the specified \p Die as having canonical + /// die, if applicable. + void markODRCanonicalDie(const DWARFDie &Die, CompileUnit &CU); + /// \defgroup FindRootDIEs Find DIEs corresponding to Address map entries. /// /// @{ @@ -778,6 +791,9 @@ private: /// Print statistics. bool Statistics = false; + /// Verify the input DWARF. + bool VerifyInputDWARF = false; + /// Skip emitting output bool NoOutput = false; @@ -795,7 +811,8 @@ private: unsigned Threads = 1; /// The accelerator table kind - AccelTableKind TheAccelTableKind = AccelTableKind::Default; + DwarfLinkerAccelTableKind TheAccelTableKind = + DwarfLinkerAccelTableKind::Default; /// Prepend path for the clang modules. std::string PrependPath; diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h index afba19ac7d42..788275782235 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h @@ -9,10 +9,10 @@ #ifndef LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H #define LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/CodeGen/DIE.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" -#include "llvm/Support/DataExtractor.h" namespace llvm { @@ -74,6 +74,12 @@ public: /// Does DIE transitively refer an incomplete decl? bool Incomplete : 1; + + /// Is DIE in the clang module scope? + bool InModuleScope : 1; + + /// Is ODR marking done? + bool ODRMarkingDone : 1; }; CompileUnit(DWARFUnit &OrigUnit, unsigned ID, bool CanUseODR, diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h index d2274488e85f..fb02b0fc1b4d 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h @@ -14,14 +14,15 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/NonRelocatableStringpool.h" -#include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include <atomic> namespace llvm { +class CompileUnit; struct DeclMapInfo; /// Small helper that resolves and caches file paths. This helps reduce the @@ -91,6 +92,10 @@ public: bool setLastSeenDIE(CompileUnit &U, const DWARFDie &Die); + void setHasCanonicalDIE() { HasCanonicalDIE = true; } + + bool hasCanonicalDIE() const { return HasCanonicalDIE; } + uint32_t getCanonicalDIEOffset() const { return CanonicalDIEOffset; } void setCanonicalDIEOffset(uint32_t Offset) { CanonicalDIEOffset = Offset; } @@ -112,7 +117,8 @@ private: const DeclContext &Parent; DWARFDie LastSeenDIE; uint32_t LastSeenCompileUnitID = 0; - uint32_t CanonicalDIEOffset = 0; + std::atomic<uint32_t> CanonicalDIEOffset = {0}; + bool HasCanonicalDIE = false; }; /// This class gives a tree-like API to the DenseMap that stores the diff --git a/llvm/include/llvm/DWARFLinker/DWARFStreamer.h b/llvm/include/llvm/DWARFLinker/DWARFStreamer.h index fc8c59904cfb..003fe548252a 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFStreamer.h +++ b/llvm/include/llvm/DWARFLinker/DWARFStreamer.h @@ -10,7 +10,6 @@ #define LLVM_DWARFLINKER_DWARFSTREAMER_H #include "llvm/BinaryFormat/Swift.h" -#include "llvm/CodeGen/AccelTable.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/DWARFLinker/DWARFLinker.h" #include "llvm/MC/MCAsmInfo.h" @@ -18,9 +17,11 @@ #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" #include "llvm/Target/TargetMachine.h" namespace llvm { +template <typename DataT> class AccelTable; enum class OutputFileType { Object, diff --git a/llvm/include/llvm/DWP/DWPStringPool.h b/llvm/include/llvm/DWP/DWPStringPool.h index 9f69851f0055..1354b46f156b 100644 --- a/llvm/include/llvm/DWP/DWPStringPool.h +++ b/llvm/include/llvm/DWP/DWPStringPool.h @@ -43,7 +43,7 @@ public: auto Pair = Pool.insert(std::make_pair(Str, Offset)); if (Pair.second) { - Out.SwitchSection(Sec); + Out.switchSection(Sec); Out.emitBytes(StringRef(Str, Length)); Offset += Length; } diff --git a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h index 5a91682e9bd4..d474173973b5 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h @@ -11,7 +11,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" #include "llvm/DebugInfo/CodeView/TypeCollection.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" diff --git a/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h b/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h index 82ef8c173bee..ef44b622d955 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h @@ -10,7 +10,7 @@ #define LLVM_DEBUGINFO_CODEVIEW_CVSYMBOLVISITOR_H #include "llvm/DebugInfo/CodeView/CVRecord.h" -#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/Error.h" namespace llvm { namespace codeview { @@ -18,12 +18,20 @@ class SymbolVisitorCallbacks; class CVSymbolVisitor { public: + struct FilterOptions { + llvm::Optional<uint32_t> SymbolOffset; + llvm::Optional<uint32_t> ParentRecursiveDepth; + llvm::Optional<uint32_t> ChildRecursiveDepth; + }; + CVSymbolVisitor(SymbolVisitorCallbacks &Callbacks); Error visitSymbolRecord(CVSymbol &Record); Error visitSymbolRecord(CVSymbol &Record, uint32_t Offset); Error visitSymbolStream(const CVSymbolArray &Symbols); Error visitSymbolStream(const CVSymbolArray &Symbols, uint32_t InitialOffset); + Error visitSymbolStreamFiltered(const CVSymbolArray &Symbols, + const FilterOptions &Filter); private: SymbolVisitorCallbacks &Callbacks; diff --git a/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h b/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h index 7538cb2c2548..7780e233cab3 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h @@ -9,14 +9,17 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_CVTYPEVISITOR_H #define LLVM_DEBUGINFO_CODEVIEW_CVTYPEVISITOR_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/Support/Error.h" namespace llvm { namespace codeview { +class TypeIndex; class TypeCollection; class TypeVisitorCallbacks; +struct CVMemberRecord; enum VisitorDataSource { VDS_BytesPresent, // The record bytes are passed into the visitation diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h index d851dea0a27f..4fbe7e835a8a 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h @@ -9,14 +9,11 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_CODEVIEWRECORDIO_H #define LLVM_DEBUGINFO_CODEVIEW_CODEVIEWRECORDIO_H -#include "llvm/ADT/APSInt.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeViewError.h" -#include "llvm/DebugInfo/CodeView/GUID.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/Error.h" @@ -26,7 +23,12 @@ namespace llvm { +template <typename T> class ArrayRef; +class APSInt; + namespace codeview { +class TypeIndex; +struct GUID; class CodeViewRecordStreamer { public: @@ -246,7 +248,7 @@ private: Optional<uint32_t> MaxLength; Optional<uint32_t> bytesRemaining(uint32_t CurrentOffset) const { - if (!MaxLength.hasValue()) + if (!MaxLength) return None; assert(CurrentOffset >= BeginOffset); diff --git a/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h index 0e2f5d90e243..0f83ae370a1e 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h @@ -12,22 +12,16 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/RecordSerialization.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/TypeRecordMapping.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryStreamWriter.h" -#include "llvm/Support/Error.h" -#include <cassert> #include <cstdint> -#include <memory> #include <vector> namespace llvm { namespace codeview { +class TypeIndex; enum class ContinuationRecordKind { FieldList, MethodOverloadList }; class ContinuationRecordBuilder { diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h index 01f83676afdf..615fd216e655 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h @@ -16,7 +16,6 @@ #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryStreamArray.h" -#include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" #include <cstdint> @@ -24,6 +23,9 @@ namespace llvm { +class BinaryStreamReader; +class BinaryStreamWriter; + namespace codeview { class DebugStringTableSubsection; diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h index 64a78a7cef21..e21873a3af8f 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h @@ -12,13 +12,14 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/Support/BinaryStreamArray.h" -#include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" #include <cstdint> #include <map> namespace llvm { +class BinaryStreamReader; +class BinaryStreamWriter; namespace codeview { class DebugCrossModuleExportsSubsectionRef final : public DebugSubsectionRef { diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h index e7683cb2a9c4..198ce4a8b4e4 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h @@ -14,7 +14,6 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/Support/BinaryStreamArray.h" -#include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -22,6 +21,8 @@ #include <vector> namespace llvm { +class BinaryStreamReader; +class BinaryStreamWriter; namespace codeview { diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h index d5cd640231f9..f2c5bf9d7c95 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h @@ -11,11 +11,15 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" -#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/BinaryStreamArray.h" +#include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" namespace llvm { +class BinaryStreamReader; +class BinaryStreamWriter; + namespace codeview { class DebugFrameDataSubsectionRef final : public DebugSubsectionRef { public: diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h index 9fd88a64873a..f9d1507af5f3 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h @@ -12,7 +12,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" -#include "llvm/DebugInfo/CodeView/Line.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/BinaryStreamReader.h" diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h index 1f8e56c5311f..68eb9e1af3bd 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h @@ -14,7 +14,6 @@ #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/DebugInfo/CodeView/Line.h" #include "llvm/Support/BinaryStreamArray.h" -#include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -22,6 +21,8 @@ #include <vector> namespace llvm { +class BinaryStreamReader; +class BinaryStreamWriter; namespace codeview { class DebugChecksumsSubsection; diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h index 2e1cd15a3956..39413bb73b58 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h @@ -10,10 +10,12 @@ #define LLVM_DEBUGINFO_CODEVIEW_DEBUGSUBSECTION_H #include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/Support/BinaryStreamWriter.h" -#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" + +#include <cstdint> namespace llvm { +class BinaryStreamWriter; namespace codeview { class DebugSubsectionRef { diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h index 151930d6d43d..fdca2ad063a1 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h @@ -9,7 +9,6 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_DEBUGSUBSECTIONVISITOR_H #define LLVM_DEBUGINFO_CODEVIEW_DEBUGSUBSECTIONVISITOR_H -#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/Support/Error.h" diff --git a/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h b/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h index 270cd4b8330c..ec874b7ca114 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h +++ b/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h @@ -12,10 +12,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/Support/ScopedPrinter.h" #include <cstdint> namespace llvm { +template <typename T> struct EnumEntry; namespace codeview { ArrayRef<EnumEntry<SymbolKind>> getSymbolTypeNames(); diff --git a/llvm/include/llvm/DebugInfo/CodeView/Formatters.h b/llvm/include/llvm/DebugInfo/CodeView/Formatters.h index 7d04a6a89bef..10683c289224 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/Formatters.h +++ b/llvm/include/llvm/DebugInfo/CodeView/Formatters.h @@ -22,6 +22,8 @@ namespace llvm { namespace codeview { +struct GUID; + namespace detail { class GuidAdapter final : public FormatAdapter<ArrayRef<uint8_t>> { diff --git a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h index 465c26ec2ce6..d592bde18bae 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -10,9 +10,9 @@ #define LLVM_DEBUGINFO_CODEVIEW_GLOBALTYPETABLEBUILDER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" #include "llvm/DebugInfo/CodeView/TypeCollection.h" #include "llvm/DebugInfo/CodeView/TypeHashing.h" diff --git a/llvm/include/llvm/DebugInfo/CodeView/Line.h b/llvm/include/llvm/DebugInfo/CodeView/Line.h index eb2aa154df1b..6918645b94d2 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/Line.h +++ b/llvm/include/llvm/DebugInfo/CodeView/Line.h @@ -9,7 +9,6 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_LINE_H #define LLVM_DEBUGINFO_CODEVIEW_LINE_H -#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/Support/Endian.h" #include <cinttypes> diff --git a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h index 0f9d5e476075..1965aab9b5cc 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h @@ -10,18 +10,18 @@ #define LLVM_DEBUGINFO_CODEVIEW_MERGINGTYPETABLEBUILDER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" #include "llvm/DebugInfo/CodeView/TypeCollection.h" -#include "llvm/DebugInfo/CodeView/TypeHashing.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/Support/Allocator.h" #include <cstdint> namespace llvm { namespace codeview { +struct LocallyHashedType; class ContinuationRecordBuilder; diff --git a/llvm/include/llvm/DebugInfo/CodeView/RecordName.h b/llvm/include/llvm/DebugInfo/CodeView/RecordName.h index 8e06be9e41e8..9078ed38d2f1 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/RecordName.h +++ b/llvm/include/llvm/DebugInfo/CodeView/RecordName.h @@ -9,11 +9,14 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_RECORDNAME_H #define LLVM_DEBUGINFO_CODEVIEW_RECORDNAME_H -#include "llvm/DebugInfo/CodeView/TypeCollection.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include <string> namespace llvm { namespace codeview { +class TypeCollection; +class TypeIndex; std::string computeTypeName(TypeCollection &Types, TypeIndex Index); StringRef getSymbolName(CVSymbol Sym); } // namespace codeview diff --git a/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h b/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h index 36c0f2fbd8fa..10248dbf646b 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h +++ b/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h @@ -9,7 +9,6 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_RECORDSERIALIZATION_H #define LLVM_DEBUGINFO_CODEVIEW_RECORDSERIALIZATION_H -#include "llvm/ADT/APSInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -18,9 +17,9 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include <cinttypes> -#include <tuple> namespace llvm { +class APSInt; namespace codeview { using llvm::support::little32_t; using llvm::support::ulittle16_t; diff --git a/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h b/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h index 22a283e785e1..50e745e5c2ab 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h +++ b/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h @@ -10,13 +10,15 @@ #define LLVM_DEBUGINFO_CODEVIEW_STRINGSANDCHECKSUMS_H #include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include <memory> namespace llvm { namespace codeview { +class DebugChecksumsSubsection; +class DebugChecksumsSubsectionRef; +class DebugStringTableSubsection; +class DebugStringTableSubsectionRef; class StringsAndChecksumsRef { public: diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h index aaeffb2446ad..c674700fac59 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h +++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h @@ -9,11 +9,13 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_SYMBOLDUMPER_H #define LLVM_DEBUGINFO_CODEVIEW_SYMBOLDUMPER_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringSet.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/SymbolDumpDelegate.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/Support/Error.h" + +#include <memory> +#include <utility> namespace llvm { class ScopedPrinter; diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h index c37f6b4d5fa7..9513e19a330a 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -196,7 +196,7 @@ struct BinaryAnnotationIterator const DecodedAnnotation &operator*() { ParseCurrentAnnotation(); - return Current.getValue(); + return *Current; } private: @@ -249,7 +249,7 @@ private: } bool ParseCurrentAnnotation() { - if (Current.hasValue()) + if (Current) return true; Next = Data; diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h index fb806c692cfd..53986f9a6db6 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h +++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h @@ -10,15 +10,17 @@ #define LLVM_DEBUGINFO_CODEVIEW_SYMBOLSERIALIZER_H #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/RecordSerialization.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/SymbolRecordMapping.h" #include "llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" +#include <array> #include <cstdint> namespace llvm { diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h b/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h index bde5a8b3ab2f..f643bc4d7451 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h @@ -34,7 +34,7 @@ public: template <typename TFunc> void ForEachRecord(TFunc Func) { Optional<TypeIndex> Next = getFirst(); - while (Next.hasValue()) { + while (Next) { TypeIndex N = *Next; Func(N, getType(N)); Next = getNext(N); diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h b/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h index 41a219ae5a7b..1fad50343e3a 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h @@ -9,16 +9,18 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEDUMPVISITOR_H #define LLVM_DEBUGINFO_CODEVIEW_TYPEDUMPVISITOR_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" namespace llvm { class ScopedPrinter; namespace codeview { +class TypeIndex; +struct CVMemberRecord; +struct MemberAttributes; class TypeCollection; diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h b/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h index 9f34d026b1ba..f49bc9b8e790 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h @@ -9,10 +9,11 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEHASHING_H #define LLVM_DEBUGINFO_CODEVIEW_TYPEHASHING_H -#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/TypeCollection.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" @@ -21,6 +22,7 @@ #include <type_traits> namespace llvm { +class raw_ostream; namespace codeview { /// A locally hashed type represents a straightforward hash code of a serialized diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h b/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h index 226a436c0930..653eafa04e0a 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -13,7 +13,6 @@ #include "llvm/Support/Endian.h" #include <cassert> #include <cinttypes> -#include <functional> namespace llvm { @@ -36,6 +35,7 @@ enum class SimpleTypeKind : uint32_t { WideCharacter = 0x0071, // wide char Character16 = 0x007a, // char16_t Character32 = 0x007b, // char32_t + Character8 = 0x007c, // char8_t SByte = 0x0068, // 8 bit signed int Byte = 0x0069, // 8 bit unsigned int diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h b/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h index f4f5835d8b57..7ef8521604fb 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h @@ -9,13 +9,13 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEINDEXDISCOVERY_H #define LLVM_DEBUGINFO_CODEVIEW_TYPEINDEXDISCOVERY_H -#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/Support/Error.h" namespace llvm { +template <typename T> class SmallVectorImpl; namespace codeview { +class TypeIndex; enum class TiRefKind { TypeRef, IndexRef }; struct TiReference { TiRefKind Kind; diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h index c6044d5138a8..ed4fc7a75624 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h @@ -10,7 +10,8 @@ #define LLVM_DEBUGINFO_CODEVIEW_TYPERECORDMAPPING_H #include "llvm/ADT/Optional.h" -#include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h" #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" #include "llvm/Support/Error.h" @@ -20,6 +21,8 @@ class BinaryStreamReader; class BinaryStreamWriter; namespace codeview { +class TypeIndex; +struct CVMemberRecord; class TypeRecordMapping : public TypeVisitorCallbacks { public: explicit TypeRecordMapping(BinaryStreamReader &Reader) : IO(Reader) {} diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h index 04d7c7b0420a..04a1e44dd809 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h @@ -10,11 +10,12 @@ #define LLVM_DEBUGINFO_CODEVIEW_TYPESTREAMMERGER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/Support/Error.h" namespace llvm { +template <typename T> class Optional; +template <typename T> class SmallVectorImpl; namespace codeview { class TypeIndex; diff --git a/llvm/include/llvm/DebugInfo/DIContext.h b/llvm/include/llvm/DebugInfo/DIContext.h index d029556c9d89..9b278b696073 100644 --- a/llvm/include/llvm/DebugInfo/DIContext.h +++ b/llvm/include/llvm/DebugInfo/DIContext.h @@ -90,6 +90,8 @@ class DIInliningInfo { public: DIInliningInfo() = default; + /// Returns the frame at `Index`. Frames are stored in bottom-up + /// (leaf-to-root) order with increasing index. const DILineInfo &getFrame(unsigned Index) const { assert(Index < Frames.size()); return Frames[Index]; @@ -112,6 +114,8 @@ struct DIGlobal { std::string Name; uint64_t Start = 0; uint64_t Size = 0; + std::string DeclFile; + uint64_t DeclLine = 0; DIGlobal() : Name(DILineInfo::BadString) {} }; @@ -151,6 +155,10 @@ struct DILineInfoSpecifier { DILineInfoSpecifier(FileLineInfoKind FLIKind = FileLineInfoKind::RawValue, FunctionNameKind FNKind = FunctionNameKind::None) : FLIKind(FLIKind), FNKind(FNKind) {} + + inline bool operator==(const DILineInfoSpecifier &RHS) const { + return FLIKind == RHS.FLIKind && FNKind == RHS.FNKind; + } }; /// This is just a helper to programmatically construct DIDumpType. @@ -233,6 +241,8 @@ public: virtual DILineInfo getLineInfoForAddress( object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; + virtual DILineInfo + getLineInfoForDataAddress(object::SectionedAddress Address) = 0; virtual DILineInfoTable getLineInfoForAddressRange( object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h index cdf3f60f88be..3887656ceef6 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h @@ -13,13 +13,13 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/Support/DataExtractor.h" #include <cassert> #include <cstddef> #include <cstdint> namespace llvm { +class DataExtractor; class DWARFFormValue; class DWARFUnit; class raw_ostream; @@ -34,7 +34,7 @@ public: AttributeSpec(dwarf::Attribute A, dwarf::Form F, Optional<uint8_t> ByteSize) : Attr(A), Form(F) { assert(!isImplicitConst()); - this->ByteSize.HasByteSize = ByteSize.hasValue(); + this->ByteSize.HasByteSize = ByteSize.has_value(); if (this->ByteSize.HasByteSize) this->ByteSize.ByteSize = *ByteSize; } diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h index 537a03ec11fc..f4d6c451cbe1 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h @@ -10,6 +10,9 @@ #define LLVM_DEBUGINFO_DWARF_DWARFADDRESSRANGE_H #include "llvm/DebugInfo/DIContext.h" +#include "llvm/Object/ObjectFile.h" +#include <algorithm> +#include <cassert> #include <cstdint> #include <tuple> #include <vector> diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h index ec5a3cd85266..d449b7bed796 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h @@ -10,10 +10,15 @@ #define LLVM_DEBUGINFO_DWARF_DWARFCOMPILEUNIT_H #include "llvm/DebugInfo/DWARF/DWARFUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" namespace llvm { +class DWARFContext; +class DWARFDebugAbbrev; +class raw_ostream; +struct DIDumpOptions; +struct DWARFSection; + class DWARFCompileUnit : public DWARFUnit { public: DWARFCompileUnit(DWARFContext &Context, const DWARFSection &Section, diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index e82faf6eeb24..bf591ed554c6 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -9,43 +9,37 @@ #ifndef LLVM_DEBUGINFO_DWARF_DWARFCONTEXT_H #define LLVM_DEBUGINFO_DWARF_DWARFCONTEXT_H -#include "llvm/ADT/MapVector.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/DIContext.h" -#include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h" -#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugAranges.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" -#include "llvm/DebugInfo/DWARF/DWARFGdbIndex.h" #include "llvm/DebugInfo/DWARF/DWARFObject.h" -#include "llvm/DebugInfo/DWARF/DWARFSection.h" -#include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" #include "llvm/Support/Host.h" #include <cstdint> -#include <deque> -#include <map> #include <memory> namespace llvm { class MCRegisterInfo; class MemoryBuffer; -class raw_ostream; +class AppleAcceleratorTable; +class DWARFCompileUnit; +class DWARFDebugAbbrev; +class DWARFDebugAranges; +class DWARFDebugFrame; +class DWARFDebugLoc; +class DWARFDebugMacro; +class DWARFDebugNames; +class DWARFGdbIndex; +class DWARFTypeUnit; +class DWARFUnitIndex; /// DWARFContext /// This data structure is the top level entity that deals with dwarf debug @@ -124,7 +118,7 @@ public: WithColor::defaultErrorHandler, std::function<void(Error)> WarningHandler = WithColor::defaultWarningHandler); - ~DWARFContext(); + ~DWARFContext() override; DWARFContext(DWARFContext &) = delete; DWARFContext &operator=(DWARFContext &) = delete; @@ -339,6 +333,10 @@ public: getLineTableForUnit(DWARFUnit *U, function_ref<void(Error)> RecoverableErrorHandler); + // Clear the line table object corresponding to a compile unit for memory + // management purpose. When it's referred to again, it'll be re-populated. + void clearLineTableForUnit(DWARFUnit *U); + DataExtractor getStringExtractor() const { return DataExtractor(DObj->getStrSection(), false, 0); } @@ -366,6 +364,8 @@ public: DILineInfo getLineInfoForAddress( object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; + DILineInfo + getLineInfoForDataAddress(object::SectionedAddress Address) override; DILineInfoTable getLineInfoForAddressRange( object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h index e1407ddd89eb..67d9ce1476dd 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h @@ -11,17 +11,14 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DIContext.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" #include <cstdint> -#include <map> #include <vector> namespace llvm { -class Error; class raw_ostream; +class DWARFDataExtractor; /// A class representing an address table as specified in DWARF v5. /// The table consists of a header followed by an array of address values from diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h index 65334b4a4976..760d8826771c 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h @@ -10,7 +10,7 @@ #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGARANGESET_H #include "llvm/ADT/iterator_range.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Support/Error.h" #include <cstdint> #include <vector> @@ -18,6 +18,7 @@ namespace llvm { class raw_ostream; +class DWARFDataExtractor; class DWARFDebugArangeSet { public: diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h index 216dd1e4defc..068674cfae5c 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h @@ -10,11 +10,13 @@ #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGARANGES_H #include "llvm/ADT/DenseSet.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include <cstdint> #include <vector> namespace llvm { +class DWARFDataExtractor; +class Error; class DWARFContext; @@ -26,7 +28,8 @@ public: private: void clear(); void extract(DWARFDataExtractor DebugArangesData, - function_ref<void(Error)> RecoverableErrorHandler); + function_ref<void(Error)> RecoverableErrorHandler, + function_ref<void(Error)> WarningHandler); /// Call appendRange multiple times and then call construct. void appendRange(uint64_t CUOffset, uint64_t LowPC, uint64_t HighPC); diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index 8167aaaeffb5..48df091412bf 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -13,7 +13,6 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFExpression.h" #include "llvm/Support/Error.h" #include <map> @@ -23,6 +22,9 @@ namespace llvm { class raw_ostream; +class DWARFDataExtractor; +class MCRegisterInfo; +struct DIDumpOptions; namespace dwarf { @@ -130,7 +132,7 @@ public: uint32_t getRegister() const { return RegNum; } int32_t getOffset() const { return Offset; } uint32_t getAddressSpace() const { - assert(Kind == RegPlusOffset && AddrSpace.hasValue()); + assert(Kind == RegPlusOffset && AddrSpace); return *AddrSpace; } int32_t getConstant() const { return Offset; } @@ -259,7 +261,7 @@ public: UnwindRow() : CFAValue(UnwindLocation::createUnspecified()) {} /// Returns true if the address is valid in this object. - bool hasAddress() const { return Address.hasValue(); } + bool hasAddress() const { return Address.has_value(); } /// Get the address for this row. /// diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h index 6bdd23900182..9befcc0c4182 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h @@ -11,12 +11,12 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include <cstdint> namespace llvm { class DWARFUnit; +class DWARFDataExtractor; /// DWARFDebugInfoEntry - A DIE with only the minimum required data. class DWARFDebugInfoEntry { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index ee15b6d4112d..86f90135f8d4 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -11,12 +11,10 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DIContext.h" -#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" -#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" -#include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h" +#include "llvm/DebugInfo/DWARF/DWARFUnit.h" #include "llvm/Support/MD5.h" #include "llvm/Support/Path.h" #include <cstdint> @@ -26,7 +24,6 @@ namespace llvm { -class DWARFUnit; class raw_ostream; class DWARFDebugLine { @@ -307,6 +304,7 @@ public: getOrParseLineTable(DWARFDataExtractor &DebugLineData, uint64_t Offset, const DWARFContext &Ctx, const DWARFUnit *U, function_ref<void(Error)> RecoverableErrorHandler); + void clearLineTable(uint64_t Offset); /// Helper to allow for parsing of an entire .debug_line section in sequence. class SectionParser { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index 1794f6649827..90e009e514d4 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -11,10 +11,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" -#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include "llvm/Support/Errc.h" #include <cstdint> @@ -22,6 +19,12 @@ namespace llvm { class DWARFUnit; class MCRegisterInfo; class raw_ostream; +class DWARFObject; +struct DIDumpOptions; +struct DWARFLocationExpression; +namespace object { +struct SectionedAddress; +} /// A single location within a location list. Entries are stored in the DWARF5 /// form even if they originally come from a DWARF<=4 location list. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h index f1768a1ddab5..d98cf9a6045a 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h @@ -12,7 +12,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" -#include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" #include <cstdint> diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h index cb347615868b..6c82bbfe74f7 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h @@ -10,16 +10,17 @@ #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGPUBTABLE_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include "llvm/DebugInfo/DWARF/DWARFObject.h" #include <cstdint> #include <vector> namespace llvm { class raw_ostream; +class DWARFDataExtractor; +class Error; /// Represents structure for holding and parsing .debug_pub* tables. class DWARFDebugPubTable { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h index 0d9f37c5610b..f4aeac1bb9db 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h @@ -10,14 +10,16 @@ #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGRANGELIST_H #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include <cassert> #include <cstdint> #include <vector> namespace llvm { class raw_ostream; +class DWARFDataExtractor; +namespace object { +struct SectionedAddress; +} class DWARFDebugRangeList { public: diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h index 2baa6493f709..13f018f53fa1 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h @@ -10,11 +10,9 @@ #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGRNGLISTS_H #include "llvm/ADT/Optional.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/DebugInfo/DIContext.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" #include "llvm/DebugInfo/DWARF/DWARFListTable.h" #include <cstdint> @@ -23,6 +21,11 @@ namespace llvm { class Error; class raw_ostream; class DWARFUnit; +class DWARFDataExtractor; +struct DIDumpOptions; +namespace object { +struct SectionedAddress; +} /// A class representing a single range list entry. struct RangeListEntry : public DWARFListEntryBase { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h index f731d440a35b..149c5ef4e493 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -18,7 +18,7 @@ #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" #include "llvm/DebugInfo/DWARF/DWARFAttribute.h" #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" +#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" #include <cassert> #include <cstdint> #include <iterator> @@ -280,6 +280,13 @@ public: /// \returns an iterator range for the attributes of the current DIE. iterator_range<attribute_iterator> attributes() const; + /// Gets the type size (in bytes) for this DIE. + /// + /// \param PointerSize the pointer size of the containing CU. + /// \returns if this is a type DIE, or this DIE contains a DW_AT_type, returns + /// the size of the type. + Optional<uint64_t> getTypeSize(uint64_t PointerSize); + class iterator; iterator begin() const; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h index b694eeacfd9d..c4d81047a4dc 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h @@ -9,16 +9,15 @@ #ifndef LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H #define LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/DebugInfo/DIContext.h" #include "llvm/Support/DataExtractor.h" namespace llvm { class DWARFUnit; +struct DIDumpOptions; class MCRegisterInfo; class raw_ostream; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 130cdb8800a9..c2c1df5b590b 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -14,12 +14,14 @@ #include "llvm/ADT/Optional.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DIContext.h" -#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/Support/DataExtractor.h" #include <cstdint> namespace llvm { class DWARFContext; +class DWARFObject; +class DWARFDataExtractor; class DWARFUnit; class raw_ostream; @@ -234,7 +236,7 @@ inline Optional<uint64_t> toUnsigned(const Optional<DWARFFormValue> &V) { /// value or the form value's encoding wasn't an unsigned constant form. inline uint64_t toUnsigned(const Optional<DWARFFormValue> &V, uint64_t Default) { - return toUnsigned(V).getValueOr(Default); + return toUnsigned(V).value_or(Default); } /// Take an optional DWARFFormValue and try to extract an reference. @@ -256,7 +258,7 @@ inline Optional<uint64_t> toReference(const Optional<DWARFFormValue> &V) { /// value or the form value's encoding wasn't a reference form. inline uint64_t toReference(const Optional<DWARFFormValue> &V, uint64_t Default) { - return toReference(V).getValueOr(Default); + return toReference(V).value_or(Default); } /// Take an optional DWARFFormValue and try to extract an signed constant. @@ -277,7 +279,7 @@ inline Optional<int64_t> toSigned(const Optional<DWARFFormValue> &V) { /// \returns the extracted signed integer value or Default if the V doesn't /// have a value or the form value's encoding wasn't a signed integer form. inline int64_t toSigned(const Optional<DWARFFormValue> &V, int64_t Default) { - return toSigned(V).getValueOr(Default); + return toSigned(V).value_or(Default); } /// Take an optional DWARFFormValue and try to extract an address. @@ -305,7 +307,7 @@ toSectionedAddress(const Optional<DWARFFormValue> &V) { /// \returns the extracted address value or Default if the V doesn't have a /// value or the form value's encoding wasn't an address form. inline uint64_t toAddress(const Optional<DWARFFormValue> &V, uint64_t Default) { - return toAddress(V).getValueOr(Default); + return toAddress(V).value_or(Default); } /// Take an optional DWARFFormValue and try to extract an section offset. @@ -327,7 +329,7 @@ inline Optional<uint64_t> toSectionOffset(const Optional<DWARFFormValue> &V) { /// have a value or the form value's encoding wasn't a section offset form. inline uint64_t toSectionOffset(const Optional<DWARFFormValue> &V, uint64_t Default) { - return toSectionOffset(V).getValueOr(Default); + return toSectionOffset(V).value_or(Default); } /// Take an optional DWARFFormValue and try to extract block data. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h index 38cd42ddb883..6b23c4e57d95 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h @@ -11,13 +11,13 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/DataExtractor.h" #include <cstdint> #include <utility> namespace llvm { class raw_ostream; +class DataExtractor; class DWARFGdbIndex { uint32_t Version; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h index 515623cedc94..84c8d71b04fc 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h @@ -14,7 +14,6 @@ #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" -#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include <cstdint> #include <map> diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h index 3add711943d0..fef59c5e95f8 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h @@ -10,6 +10,7 @@ #define LLVM_DEBUGINFO_DWARF_DWARFRELOCMAP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/Object/RelocationResolver.h" #include <cstdint> diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h new file mode 100644 index 000000000000..e05271740e61 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h @@ -0,0 +1,67 @@ +//===- DWARFTypePrinter.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_DWARF_DWARFTYPEPRINTER_H +#define LLVM_DEBUGINFO_DWARF_DWARFTYPEPRINTER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/DebugInfo/DWARF/DWARFDie.h" + +#include <string> + +namespace llvm { + +class raw_ostream; + +// FIXME: We should have pretty printers per language. Currently we print +// everything as if it was C++ and fall back to the TAG type name. +struct DWARFTypePrinter { + raw_ostream &OS; + bool Word = true; + bool EndedWithTemplate = false; + + DWARFTypePrinter(raw_ostream &OS) : OS(OS) {} + + /// Dump the name encoded in the type tag. + void appendTypeTagName(dwarf::Tag T); + + void appendArrayType(const DWARFDie &D); + + DWARFDie skipQualifiers(DWARFDie D); + + bool needsParens(DWARFDie D); + + void appendPointerLikeTypeBefore(DWARFDie D, DWARFDie Inner, StringRef Ptr); + + DWARFDie appendUnqualifiedNameBefore(DWARFDie D, + std::string *OriginalFullName = nullptr); + + void appendUnqualifiedNameAfter(DWARFDie D, DWARFDie Inner, + bool SkipFirstParamIfArtificial = false); + void appendQualifiedName(DWARFDie D); + DWARFDie appendQualifiedNameBefore(DWARFDie D); + bool appendTemplateParameters(DWARFDie D, bool *FirstParameter = nullptr); + void decomposeConstVolatile(DWARFDie &N, DWARFDie &T, DWARFDie &C, + DWARFDie &V); + void appendConstVolatileQualifierAfter(DWARFDie N); + void appendConstVolatileQualifierBefore(DWARFDie N); + + /// Recursively append the DIE type name when applicable. + void appendUnqualifiedName(DWARFDie D, + std::string *OriginalFullName = nullptr); + + void appendSubroutineNameAfter(DWARFDie D, DWARFDie Inner, + bool SkipFirstParamIfArtificial, bool Const, + bool Volatile); + void appendScopes(DWARFDie D); +}; + +} // namespace llvm + +#endif // LLVM_DEBUGINFO_DWARF_DWARFTYPEPRINTER_H diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index c95bdcbd8a43..85ec6fd86ade 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -11,12 +11,11 @@ #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" -#include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" -#include "llvm/Support/DataExtractor.h" #include <cstdint> namespace llvm { +struct DIDumpOptions; class DWARFContext; class DWARFDebugAbbrev; struct DWARFSection; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h index b96a4c19758f..9188865b4d77 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -9,28 +9,26 @@ #ifndef LLVM_DEBUGINFO_DWARF_DWARFUNIT_H #define LLVM_DEBUGINFO_DWARF_DWARFUNIT_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" +#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugRnglists.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" -#include "llvm/DebugInfo/DWARF/DWARFFormValue.h" -#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" -#include "llvm/DebugInfo/DWARF/DWARFSection.h" +#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" #include "llvm/Support/DataExtractor.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <cstdint> #include <map> #include <memory> +#include <set> #include <utility> #include <vector> @@ -40,6 +38,12 @@ class DWARFAbbreviationDeclarationSet; class DWARFContext; class DWARFDebugAbbrev; class DWARFUnit; +class DWARFDebugRangeList; +class DWARFLocationTable; +class DWARFObject; +class raw_ostream; +struct DIDumpOptions; +struct DWARFSection; /// Base class describing the header of any kind of "unit." Some information /// is specific to certain unit types. We separate this class out so we can @@ -238,6 +242,11 @@ class DWARFUnit { /// std::map::upper_bound for address range lookup. std::map<uint64_t, std::pair<uint64_t, DWARFDie>> AddrDieMap; + /// Map from the location (interpreted DW_AT_location) of a DW_TAG_variable, + /// to the end address and the corresponding DIE. + std::map<uint64_t, std::pair<uint64_t, DWARFDie>> VariableDieMap; + DenseSet<uint64_t> RootsParsedForVariables; + using die_iterator_range = iterator_range<std::vector<DWARFDebugInfoEntry>::iterator>; @@ -320,6 +329,9 @@ public: /// Recursively update address to Die map. void updateAddressDieMap(DWARFDie Die); + /// Recursively update address to variable Die map. + void updateVariableDieMap(DWARFDie Die); + void setRangesSection(const DWARFSection *RS, uint64_t Base) { RangeSection = RS; RangeSectionBase = Base; @@ -434,6 +446,10 @@ public: /// cleared. DWARFDie getSubroutineForAddress(uint64_t Address); + /// Returns variable DIE for the address provided. The pointer is alive as + /// long as parsed compile unit DIEs are not cleared. + DWARFDie getVariableForAddress(uint64_t Address); + /// getInlinedChainForAddress - fetches inlined chain for a given address. /// Returns empty chain if there is no subprogram containing address. The /// chain is valid as long as parsed compile unit DIEs are not cleared. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h index edea59e474cf..b5e191ba7def 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -11,13 +11,13 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/DataExtractor.h" #include <cstdint> #include <memory> namespace llvm { class raw_ostream; +class DataExtractor; /// The enum of section identifiers to be used in internal interfaces. /// @@ -64,6 +64,25 @@ enum DWARFSectionKind { DW_SECT_EXT_MACINFO = 10, }; +inline const char *toString(DWARFSectionKind Kind) { + switch (Kind) { + case DW_SECT_EXT_unknown: + return "Unknown DW_SECT value 0"; +#define STRINGIZE(X) #X +#define HANDLE_DW_SECT(ID, NAME) \ + case DW_SECT_##NAME: \ + return "DW_SECT_" STRINGIZE(NAME); +#include "llvm/BinaryFormat/Dwarf.def" + case DW_SECT_EXT_TYPES: + return "DW_SECT_TYPES"; + case DW_SECT_EXT_LOC: + return "DW_SECT_LOC"; + case DW_SECT_EXT_MACINFO: + return "DW_SECT_MACINFO"; + } + llvm_unreachable("unknown DWARFSectionKind"); +} + /// Convert the internal value for a section kind to an on-disk value. /// /// The conversion depends on the version of the index section. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h index 505686bfbf59..1f1ebe943238 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h @@ -12,9 +12,9 @@ #include "llvm/ADT/Optional.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h" +#include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" -#include "llvm/DebugInfo/DWARF/DWARFUnit.h" #include <cstdint> #include <map> #include <set> @@ -22,13 +22,14 @@ namespace llvm { class raw_ostream; struct DWARFAddressRange; +class DWARFUnit; +class DWARFUnitVector; struct DWARFAttribute; class DWARFContext; class DWARFDataExtractor; class DWARFDebugAbbrev; class DataExtractor; struct DWARFSection; -class DWARFUnit; /// A class that verifies DWARF debug information given a DWARF Context. class DWARFVerifier { @@ -151,12 +152,15 @@ private: /// section. /// /// \param S The DWARF Section to verify. - /// \param SectionKind The object-file section kind that S comes from. /// /// \returns The number of errors that occurred during verification. unsigned verifyUnitSection(const DWARFSection &S); unsigned verifyUnits(const DWARFUnitVector &Units); + unsigned verifyIndexes(const DWARFObject &DObj); + unsigned verifyIndex(StringRef Name, DWARFSectionKind SectionKind, + StringRef Index); + /// Verifies that a call site entry is nested within a subprogram with a /// DW_AT_call attribute. /// @@ -301,6 +305,24 @@ public: /// \returns true if all sections verify successfully, false otherwise. bool handleDebugInfo(); + /// Verify the information in the .debug_cu_index section. + /// + /// Any errors are reported to the stream that was this object was + /// constructed with. + /// + /// \returns true if the .debug_cu_index verifies successfully, false + /// otherwise. + bool handleDebugCUIndex(); + + /// Verify the information in the .debug_tu_index section. + /// + /// Any errors are reported to the stream that was this object was + /// constructed with. + /// + /// \returns true if the .debug_tu_index verifies successfully, false + /// otherwise. + bool handleDebugTUIndex(); + /// Verify the information in the .debug_line section. /// /// Any errors are reported to the stream that was this object was diff --git a/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h b/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h index 32fc54b14796..b8d7199f2d87 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h +++ b/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h @@ -10,7 +10,7 @@ #define LLVM_DEBUGINFO_GSYM_DWARFTRANSFORMER_H #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/DebugInfo/GSYM/ExtractRanges.h" #include "llvm/Support/Error.h" namespace llvm { diff --git a/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h b/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h new file mode 100644 index 000000000000..9a6568719875 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h @@ -0,0 +1,81 @@ +//===- ExtractRanges.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_EXTRACTRANGES_H +#define LLVM_DEBUGINFO_GSYM_EXTRACTRANGES_H + +#include "llvm/ADT/AddressRanges.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include <stdint.h> +#include <vector> + +#define HEX8(v) llvm::format_hex(v, 4) +#define HEX16(v) llvm::format_hex(v, 6) +#define HEX32(v) llvm::format_hex(v, 10) +#define HEX64(v) llvm::format_hex(v, 18) + +namespace llvm { +class DataExtractor; +class raw_ostream; + +namespace gsym { + +class FileWriter; + +/// AddressRange objects are encoded and decoded to be relative to a base +/// address. This will be the FunctionInfo's start address if the AddressRange +/// is directly contained in a FunctionInfo, or a base address of the +/// containing parent AddressRange or AddressRanges. This allows address +/// ranges to be efficiently encoded using ULEB128 encodings as we encode the +/// offset and size of each range instead of full addresses. This also makes +/// encoded addresses easy to relocate as we just need to relocate one base +/// address. +/// @{ +AddressRange decodeRange(DataExtractor &Data, uint64_t BaseAddr, + uint64_t &Offset); +void encodeRange(const AddressRange &Range, FileWriter &O, uint64_t BaseAddr); +/// @} + +/// Skip an address range object in the specified data a the specified +/// offset. +/// +/// \param Data The binary stream to read the data from. +/// +/// \param Offset The byte offset within \a Data. +void skipRange(DataExtractor &Data, uint64_t &Offset); + +/// Address ranges are decoded and encoded to be relative to a base address. +/// See the AddressRange comment for the encode and decode methods for full +/// details. +/// @{ +void decodeRanges(AddressRanges &Ranges, DataExtractor &Data, uint64_t BaseAddr, + uint64_t &Offset); +void encodeRanges(const AddressRanges &Ranges, FileWriter &O, + uint64_t BaseAddr); +/// @} + +/// Skip an address range object in the specified data a the specified +/// offset. +/// +/// \param Data The binary stream to read the data from. +/// +/// \param Offset The byte offset within \a Data. +/// +/// \returns The number of address ranges that were skipped. +uint64_t skipRanges(DataExtractor &Data, uint64_t &Offset); + +} // namespace gsym + +raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R); + +raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR); + +} // namespace llvm + +#endif // LLVM_DEBUGINFO_GSYM_EXTRACTRANGES_H diff --git a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h index 552337f54390..fb48f7f9a93c 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h +++ b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h @@ -10,10 +10,10 @@ #define LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/GSYM/ExtractRanges.h" #include "llvm/DebugInfo/GSYM/InlineInfo.h" #include "llvm/DebugInfo/GSYM/LineTable.h" #include "llvm/DebugInfo/GSYM/LookupResult.h" -#include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/DebugInfo/GSYM/StringTable.h" #include <cstdint> #include <tuple> @@ -102,9 +102,7 @@ struct FunctionInfo { /// debug info, we might end up with multiple FunctionInfo objects for the /// same range and we need to be able to tell which one is the better object /// to use. - bool hasRichInfo() const { - return OptLineTable.hasValue() || Inline.hasValue(); - } + bool hasRichInfo() const { return OptLineTable || Inline; } /// Query if a FunctionInfo object is valid. /// @@ -170,12 +168,9 @@ struct FunctionInfo { uint64_t FuncAddr, uint64_t Addr); - uint64_t startAddress() const { return Range.Start; } - uint64_t endAddress() const { return Range.End; } + uint64_t startAddress() const { return Range.start(); } + uint64_t endAddress() const { return Range.end(); } uint64_t size() const { return Range.size(); } - void setStartAddress(uint64_t Addr) { Range.Start = Addr; } - void setEndAddress(uint64_t Addr) { Range.End = Addr; } - void setSize(uint64_t Size) { Range.End = Range.Start + Size; } void clear() { Range = {0, 0}; @@ -203,8 +198,8 @@ inline bool operator<(const FunctionInfo &LHS, const FunctionInfo &RHS) { return LHS.Range < RHS.Range; // Then sort by inline - if (LHS.Inline.hasValue() != RHS.Inline.hasValue()) - return RHS.Inline.hasValue(); + if (LHS.Inline.has_value() != RHS.Inline.has_value()) + return RHS.Inline.has_value(); return LHS.OptLineTable < RHS.OptLineTable; } diff --git a/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h b/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h index 872ccd4a0b6a..29ad1c18e295 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h +++ b/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h @@ -14,11 +14,11 @@ #include <mutex> #include <thread> +#include "llvm/ADT/AddressRanges.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/DebugInfo/GSYM/FileEntry.h" #include "llvm/DebugInfo/GSYM/FunctionInfo.h" -#include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" diff --git a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h index 9bcfa5935180..80385116598a 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h +++ b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h @@ -10,14 +10,13 @@ #define LLVM_DEBUGINFO_GSYM_INLINEINFO_H #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/GSYM/ExtractRanges.h" #include "llvm/DebugInfo/GSYM/LineEntry.h" #include "llvm/DebugInfo/GSYM/LookupResult.h" -#include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/Support/Error.h" #include <stdint.h> #include <vector> - namespace llvm { class raw_ostream; diff --git a/llvm/include/llvm/DebugInfo/GSYM/LineEntry.h b/llvm/include/llvm/DebugInfo/GSYM/LineEntry.h index b4e7587fc5ee..e68624b21929 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/LineEntry.h +++ b/llvm/include/llvm/DebugInfo/GSYM/LineEntry.h @@ -9,7 +9,7 @@ #ifndef LLVM_DEBUGINFO_GSYM_LINEENTRY_H #define LLVM_DEBUGINFO_GSYM_LINEENTRY_H -#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/DebugInfo/GSYM/ExtractRanges.h" namespace llvm { namespace gsym { diff --git a/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h index 3dabbce32bb2..44e58f522002 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h +++ b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h @@ -9,8 +9,8 @@ #ifndef LLVM_DEBUGINFO_GSYM_LOOKUPRESULT_H #define LLVM_DEBUGINFO_GSYM_LOOKUPRESULT_H +#include "llvm/ADT/AddressRanges.h" #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/GSYM/Range.h" #include <inttypes.h> #include <vector> diff --git a/llvm/include/llvm/DebugInfo/GSYM/Range.h b/llvm/include/llvm/DebugInfo/GSYM/Range.h deleted file mode 100644 index 36ad95602d14..000000000000 --- a/llvm/include/llvm/DebugInfo/GSYM/Range.h +++ /dev/null @@ -1,130 +0,0 @@ -//===- Range.h --------------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_GSYM_RANGE_H -#define LLVM_DEBUGINFO_GSYM_RANGE_H - -#include "llvm/ADT/Optional.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" -#include <stdint.h> -#include <vector> - -#define HEX8(v) llvm::format_hex(v, 4) -#define HEX16(v) llvm::format_hex(v, 6) -#define HEX32(v) llvm::format_hex(v, 10) -#define HEX64(v) llvm::format_hex(v, 18) - -namespace llvm { -class DataExtractor; -class raw_ostream; - -namespace gsym { - -class FileWriter; - -/// A class that represents an address range. The range is specified using -/// a start and an end address. -struct AddressRange { - uint64_t Start; - uint64_t End; - AddressRange() : Start(0), End(0) {} - AddressRange(uint64_t S, uint64_t E) : Start(S), End(E) {} - uint64_t size() const { return End - Start; } - bool contains(uint64_t Addr) const { return Start <= Addr && Addr < End; } - bool intersects(const AddressRange &R) const { - return Start < R.End && R.Start < End; - } - - bool operator==(const AddressRange &R) const { - return Start == R.Start && End == R.End; - } - bool operator!=(const AddressRange &R) const { - return !(*this == R); - } - bool operator<(const AddressRange &R) const { - return std::make_pair(Start, End) < std::make_pair(R.Start, R.End); - } - /// AddressRange objects are encoded and decoded to be relative to a base - /// address. This will be the FunctionInfo's start address if the AddressRange - /// is directly contained in a FunctionInfo, or a base address of the - /// containing parent AddressRange or AddressRanges. This allows address - /// ranges to be efficiently encoded using ULEB128 encodings as we encode the - /// offset and size of each range instead of full addresses. This also makes - /// encoded addresses easy to relocate as we just need to relocate one base - /// address. - /// @{ - void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); - void encode(FileWriter &O, uint64_t BaseAddr) const; - /// @} - - /// Skip an address range object in the specified data a the specified - /// offset. - /// - /// \param Data The binary stream to read the data from. - /// - /// \param Offset The byte offset within \a Data. - static void skip(DataExtractor &Data, uint64_t &Offset); -}; - -raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R); - -/// The AddressRanges class helps normalize address range collections. -/// This class keeps a sorted vector of AddressRange objects and can perform -/// insertions and searches efficiently. The address ranges are always sorted -/// and never contain any invalid or empty address ranges. This allows us to -/// emit address ranges into the GSYM file efficiently. Intersecting address -/// ranges are combined during insertion so that we can emit the most compact -/// representation for address ranges when writing to disk. -class AddressRanges { -protected: - using Collection = std::vector<AddressRange>; - Collection Ranges; -public: - void clear() { Ranges.clear(); } - bool empty() const { return Ranges.empty(); } - bool contains(uint64_t Addr) const; - bool contains(AddressRange Range) const; - Optional<AddressRange> getRangeThatContains(uint64_t Addr) const; - void insert(AddressRange Range); - size_t size() const { return Ranges.size(); } - bool operator==(const AddressRanges &RHS) const { - return Ranges == RHS.Ranges; - } - const AddressRange &operator[](size_t i) const { - assert(i < Ranges.size()); - return Ranges[i]; - } - Collection::const_iterator begin() const { return Ranges.begin(); } - Collection::const_iterator end() const { return Ranges.end(); } - - /// Address ranges are decoded and encoded to be relative to a base address. - /// See the AddressRange comment for the encode and decode methods for full - /// details. - /// @{ - void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); - void encode(FileWriter &O, uint64_t BaseAddr) const; - /// @} - - /// Skip an address range object in the specified data a the specified - /// offset. - /// - /// \param Data The binary stream to read the data from. - /// - /// \param Offset The byte offset within \a Data. - /// - /// \returns The number of address ranges that were skipped. - static uint64_t skip(DataExtractor &Data, uint64_t &Offset); -}; - -raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR); - -} // namespace gsym -} // namespace llvm - -#endif // LLVM_DEBUGINFO_GSYM_RANGE_H diff --git a/llvm/include/llvm/DebugInfo/GSYM/StringTable.h b/llvm/include/llvm/DebugInfo/GSYM/StringTable.h index d920335d373e..d9c9ede91be5 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/StringTable.h +++ b/llvm/include/llvm/DebugInfo/GSYM/StringTable.h @@ -10,7 +10,7 @@ #define LLVM_DEBUGINFO_GSYM_STRINGTABLE_H #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/DebugInfo/GSYM/ExtractRanges.h" #include <stdint.h> namespace llvm { diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h b/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h index 1a03d42ded92..2ac18a8efaba 100644 --- a/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h +++ b/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h @@ -11,7 +11,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" -#include "llvm/DebugInfo/MSF/MSFCommon.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" #include <cstdint> @@ -22,6 +22,8 @@ namespace llvm { class FileBufferByteStream; namespace msf { +struct MSFLayout; + class MSFBuilder { public: /// Create a new `MSFBuilder`. diff --git a/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h b/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h index bfa67d39bc76..6cd5c8d1d668 100644 --- a/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h +++ b/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h @@ -9,6 +9,7 @@ #ifndef LLVM_DEBUGINFO_PDB_IPDBENUMCHILDREN_H #define LLVM_DEBUGINFO_PDB_IPDBENUMCHILDREN_H +#include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" #include <cassert> #include <cstdint> #include <memory> diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h index 70ef4d058082..1ecae5c32509 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h @@ -10,16 +10,16 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_DBIMODULEDESCRIPTOR_H #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" #include <cstdint> namespace llvm { +template <typename T> struct VarStreamArrayExtractor; namespace pdb { - +struct ModuleInfoHeader; +struct SectionContrib; class DbiModuleDescriptor { friend class DbiStreamBuilder; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h index 8a49f46320b0..287f319e01b0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h @@ -9,13 +9,12 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_DBIMODULEDESCRIPTORBUILDER_H #define LLVM_DEBUGINFO_PDB_NATIVE_DBIMODULEDESCRIPTORBUILDER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" #include <cstdint> #include <string> @@ -23,9 +22,8 @@ namespace llvm { class BinaryStreamWriter; - namespace codeview { -class DebugSubsectionRecordBuilder; +class DebugSubsection; } namespace msf { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h index 0bdb27a0a991..3f60130f5752 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h @@ -9,14 +9,10 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_DBISTREAM_H #define LLVM_DEBUGINFO_PDB_NATIVE_DBISTREAM_H -#include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" -#include "llvm/DebugInfo/MSF/MappedBlockStream.h" -#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleList.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/BinaryStreamRef.h" @@ -24,13 +20,19 @@ #include "llvm/Support/Error.h" namespace llvm { +class BinaryStream; namespace object { struct FpoData; struct coff_section; } - +namespace msf { +class MappedBlockStream; +} namespace pdb { -class DbiStreamBuilder; +struct DbiStreamHeader; +struct SecMapEntry; +struct SectionContrib2; +struct SectionContrib; class PDBFile; class ISectionContribVisitor; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index ef441d433040..2f99aa942a05 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -10,35 +10,33 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_DBISTREAMBUILDER_H #include "llvm/ADT/Optional.h" -#include "llvm/ADT/StringSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/COFF.h" +#include "llvm/Object/COFF.h" +#include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" -#include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/BinaryStreamReader.h" -#include "llvm/Support/Endian.h" +#include "llvm/Support/BinaryStreamRef.h" namespace llvm { + +class BinaryStreamWriter; namespace codeview { struct FrameData; } namespace msf { class MSFBuilder; -} -namespace object { -struct coff_section; -struct FpoData; +struct MSFLayout; } namespace pdb { -class DbiStream; -struct DbiStreamHeader; class DbiModuleDescriptorBuilder; -class PDBFile; class DbiStreamBuilder { public: @@ -134,7 +132,7 @@ private: std::vector<SecMapEntry> SectionMap; std::array<Optional<DebugStream>, (int)DbgHeaderType::Max> DbgStreams; }; -} +} // namespace pdb } #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/EnumTables.h b/llvm/include/llvm/DebugInfo/PDB/Native/EnumTables.h index 60cd494639c1..dcc67f1e4a8c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/EnumTables.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/EnumTables.h @@ -10,9 +10,9 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_ENUMTABLES_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/Support/ScopedPrinter.h" namespace llvm { +template <typename T> struct EnumEntry; namespace pdb { ArrayRef<EnumEntry<uint16_t>> getOMFSegMapDescFlagNames(); } diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h new file mode 100644 index 000000000000..ed745eaf9727 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h @@ -0,0 +1,133 @@ +//===- FormatUtil.h ------------------------------------------- *- C++ --*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H +#define LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/FormatAdapters.h" +#include "llvm/Support/FormatVariadic.h" + +#include <string> +#include <type_traits> + +namespace llvm { +namespace pdb { + +#define PUSH_MASKED_FLAG(Enum, Mask, TheOpt, Value, Text) \ + if (Enum::TheOpt == (Value & Mask)) \ + Opts.push_back(Text); + +#define PUSH_FLAG(Enum, TheOpt, Value, Text) \ + PUSH_MASKED_FLAG(Enum, Enum::TheOpt, TheOpt, Value, Text) + +#define RETURN_CASE(Enum, X, Ret) \ + case Enum::X: \ + return Ret; + +template <typename T> std::string formatUnknownEnum(T Value) { + return formatv("unknown ({0})", static_cast<std::underlying_type_t<T>>(Value)) + .str(); +} + +std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset); + +enum class CharacteristicStyle { + HeaderDefinition, // format as windows header definition + Descriptive, // format as human readable words +}; +std::string formatSectionCharacteristics( + uint32_t IndentLevel, uint32_t C, uint32_t FlagsPerLine, + StringRef Separator, + CharacteristicStyle Style = CharacteristicStyle::HeaderDefinition); + +std::string typesetItemList(ArrayRef<std::string> Opts, uint32_t IndentLevel, + uint32_t GroupSize, StringRef Sep); + +std::string typesetStringList(uint32_t IndentLevel, + ArrayRef<StringRef> Strings); + +std::string formatChunkKind(codeview::DebugSubsectionKind Kind, + bool Friendly = true); +std::string formatSymbolKind(codeview::SymbolKind K); +std::string formatTypeLeafKind(codeview::TypeLeafKind K); + +/// Returns the number of digits in the given integer. +inline int NumDigits(uint64_t N) { + if (N < 10ULL) + return 1; + if (N < 100ULL) + return 2; + if (N < 1000ULL) + return 3; + if (N < 10000ULL) + return 4; + if (N < 100000ULL) + return 5; + if (N < 1000000ULL) + return 6; + if (N < 10000000ULL) + return 7; + if (N < 100000000ULL) + return 8; + if (N < 1000000000ULL) + return 9; + if (N < 10000000000ULL) + return 10; + if (N < 100000000000ULL) + return 11; + if (N < 1000000000000ULL) + return 12; + if (N < 10000000000000ULL) + return 13; + if (N < 100000000000000ULL) + return 14; + if (N < 1000000000000000ULL) + return 15; + if (N < 10000000000000000ULL) + return 16; + if (N < 100000000000000000ULL) + return 17; + if (N < 1000000000000000000ULL) + return 18; + if (N < 10000000000000000000ULL) + return 19; + return 20; +} + +namespace detail { +template <typename T> +struct EndianAdapter final + : public FormatAdapter<support::detail::packed_endian_specific_integral< + T, support::little, support::unaligned>> { + using EndianType = + support::detail::packed_endian_specific_integral<T, support::little, + support::unaligned>; + + explicit EndianAdapter(EndianType &&Item) + : FormatAdapter<EndianType>(std::move(Item)) {} + + void format(llvm::raw_ostream &Stream, StringRef Style) override { + format_provider<T>::format(static_cast<T>(this->Item), Stream, Style); + } +}; +} // namespace detail + +template <typename T> +detail::EndianAdapter<T> +fmtle(support::detail::packed_endian_specific_integral<T, support::little, + support::unaligned> + Value) { + return detail::EndianAdapter<T>(std::move(Value)); +} +} // namespace pdb +} // namespace llvm +#endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h index 9530a15849d5..28a72c887f25 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h @@ -10,18 +10,20 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_GSISTREAMBUILDER_H #include "llvm/ADT/DenseSet.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/BinaryItemStream.h" #include "llvm/Support/BinaryStreamRef.h" -#include "llvm/Support/BinaryStreamWriter.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" namespace llvm { +namespace codeview { +class ConstantSym; +class DataSym; +class ProcRefSym; +} // namespace codeview +template <typename T> struct BinaryItemTraits; template <> struct BinaryItemTraits<codeview::CVSymbol> { static size_t length(const codeview::CVSymbol &Item) { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h index 2b74babd6ab9..2988bef4a75b 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h @@ -10,18 +10,18 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_GLOBALSSTREAM_H #include "llvm/ADT/iterator.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" -#include "llvm/DebugInfo/MSF/MappedBlockStream.h" -#include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" namespace llvm { +class BinaryStreamReader; +namespace msf { +class MappedBlockStream; +} namespace pdb { -class DbiStream; -class PDBFile; class SymbolStream; /// Iterator over hash records producing symbol record offsets. Abstracts away @@ -81,7 +81,7 @@ private: GSIHashTable GlobalsTable; std::unique_ptr<msf::MappedBlockStream> Stream; }; -} +} // namespace pdb } #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h index 474bd796b2b3..7924cffd640f 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h @@ -23,9 +23,6 @@ namespace llvm { -class BinaryStreamReader; -class BinaryStreamWriter; - namespace pdb { Error readSparseBitVector(BinaryStreamReader &Stream, SparseBitVector<> &V); diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h index 67db92b64913..625bab6a4378 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h @@ -9,22 +9,18 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_INFOSTREAM_H #define LLVM_DEBUGINFO_PDB_NATIVE_INFOSTREAM_H -#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/StringMap.h" #include "llvm/DebugInfo/CodeView/GUID.h" -#include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" +#include "llvm/Support/BinaryStream.h" +#include "llvm/Support/BinaryStreamRef.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" namespace llvm { namespace pdb { -class InfoStreamBuilder; -class PDBFile; - +struct InfoStreamHeader; class InfoStream { friend class InfoStreamBuilder; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h index 4952173c5873..2d5088a3bd42 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h @@ -12,19 +12,17 @@ #include "llvm/ADT/Optional.h" #include "llvm/Support/Error.h" -#include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h" -#include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/CodeView/GUID.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { class WritableBinaryStreamRef; namespace msf { class MSFBuilder; +struct MSFLayout; } namespace pdb { -class PDBFile; class NamedStreamMap; class InfoStreamBuilder { @@ -70,7 +68,7 @@ private: NamedStreamMap &NamedStreams; }; -} +} // namespace pdb } #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h index b2ba81a88254..259c924d9d7c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h @@ -9,15 +9,14 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_INJECTEDSOURCESTREAM_H #define LLVM_DEBUGINFO_PDB_NATIVE_INJECTEDSOURCESTREAM_H +#include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/HashTable.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/Support/Error.h" namespace llvm { -namespace msf { -class MappedBlockStream; -} namespace pdb { +struct SrcHeaderBlockEntry; +struct SrcHeaderBlockHeader; class PDBStringTable; class InjectedSourceStream { @@ -38,6 +37,6 @@ private: HashTable<SrcHeaderBlockEntry> InjectedSourceTable; }; } -} +} // namespace llvm #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h new file mode 100644 index 000000000000..c0d722960540 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h @@ -0,0 +1,231 @@ +//===- InputFile.h -------------------------------------------- *- C++ --*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_INPUTFILE_H +#define LLVM_DEBUGINFO_PDB_NATIVE_INPUTFILE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/iterator.h" +#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" +#include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" +#include "llvm/DebugInfo/PDB/Native/LinePrinter.h" +#include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace codeview { +class LazyRandomTypeCollection; +} +namespace object { +class COFFObjectFile; +} // namespace object + +namespace pdb { +class InputFile; +class LinePrinter; +class PDBFile; +class NativeSession; +class SymbolGroupIterator; +class SymbolGroup; + +class InputFile { + InputFile(); + + std::unique_ptr<NativeSession> PdbSession; + object::OwningBinary<object::Binary> CoffObject; + std::unique_ptr<MemoryBuffer> UnknownFile; + PointerUnion<PDBFile *, object::COFFObjectFile *, MemoryBuffer *> PdbOrObj; + + using TypeCollectionPtr = std::unique_ptr<codeview::LazyRandomTypeCollection>; + + TypeCollectionPtr Types; + TypeCollectionPtr Ids; + + enum TypeCollectionKind { kTypes, kIds }; + codeview::LazyRandomTypeCollection & + getOrCreateTypeCollection(TypeCollectionKind Kind); + +public: + InputFile(PDBFile *Pdb) { PdbOrObj = Pdb; } + InputFile(object::COFFObjectFile *Obj) { PdbOrObj = Obj; } + InputFile(MemoryBuffer *Buffer) { PdbOrObj = Buffer; } + ~InputFile(); + InputFile(InputFile &&Other) = default; + + static Expected<InputFile> open(StringRef Path, + bool AllowUnknownFile = false); + + PDBFile &pdb(); + const PDBFile &pdb() const; + object::COFFObjectFile &obj(); + const object::COFFObjectFile &obj() const; + MemoryBuffer &unknown(); + const MemoryBuffer &unknown() const; + + StringRef getFilePath() const; + + bool hasTypes() const; + bool hasIds() const; + + codeview::LazyRandomTypeCollection &types(); + codeview::LazyRandomTypeCollection &ids(); + + iterator_range<SymbolGroupIterator> symbol_groups(); + SymbolGroupIterator symbol_groups_begin(); + SymbolGroupIterator symbol_groups_end(); + + bool isPdb() const; + bool isObj() const; + bool isUnknown() const; +}; + +class SymbolGroup { + friend class SymbolGroupIterator; + +public: + explicit SymbolGroup(InputFile *File, uint32_t GroupIndex = 0); + + Expected<StringRef> getNameFromStringTable(uint32_t Offset) const; + Expected<StringRef> getNameFromChecksums(uint32_t Offset) const; + + void formatFromFileName(LinePrinter &Printer, StringRef File, + bool Append = false) const; + + void formatFromChecksumsOffset(LinePrinter &Printer, uint32_t Offset, + bool Append = false) const; + + StringRef name() const; + + codeview::DebugSubsectionArray getDebugSubsections() const { + return Subsections; + } + const ModuleDebugStreamRef &getPdbModuleStream() const; + + const InputFile &getFile() const { return *File; } + InputFile &getFile() { return *File; } + + bool hasDebugStream() const { return DebugStream != nullptr; } + +private: + void initializeForPdb(uint32_t Modi); + void updatePdbModi(uint32_t Modi); + void updateDebugS(const codeview::DebugSubsectionArray &SS); + + void rebuildChecksumMap(); + InputFile *File = nullptr; + StringRef Name; + codeview::DebugSubsectionArray Subsections; + std::shared_ptr<ModuleDebugStreamRef> DebugStream; + codeview::StringsAndChecksumsRef SC; + StringMap<codeview::FileChecksumEntry> ChecksumsByFile; +}; + +class SymbolGroupIterator + : public iterator_facade_base<SymbolGroupIterator, + std::forward_iterator_tag, SymbolGroup> { +public: + SymbolGroupIterator(); + explicit SymbolGroupIterator(InputFile &File); + SymbolGroupIterator(const SymbolGroupIterator &Other) = default; + SymbolGroupIterator &operator=(const SymbolGroupIterator &R) = default; + + const SymbolGroup &operator*() const; + SymbolGroup &operator*(); + + bool operator==(const SymbolGroupIterator &R) const; + SymbolGroupIterator &operator++(); + +private: + void scanToNextDebugS(); + bool isEnd() const; + + uint32_t Index = 0; + Optional<object::section_iterator> SectionIter; + SymbolGroup Value; +}; + +Expected<ModuleDebugStreamRef> +getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index); +Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File, + uint32_t Index); + +bool shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group, + const FilterOptions &Filters); + +// TODO: Change these callbacks to be function_refs (de-templatify them). +template <typename CallbackT> +Error iterateOneModule(InputFile &File, const PrintScope &HeaderScope, + const SymbolGroup &SG, uint32_t Modi, + CallbackT Callback) { + HeaderScope.P.formatLine( + "Mod {0:4} | `{1}`: ", + fmt_align(Modi, AlignStyle::Right, HeaderScope.LabelWidth), SG.name()); + + AutoIndent Indent(HeaderScope); + return Callback(Modi, SG); +} + +template <typename CallbackT> +Error iterateSymbolGroups(InputFile &Input, const PrintScope &HeaderScope, + CallbackT Callback) { + AutoIndent Indent(HeaderScope); + + FilterOptions Filters = HeaderScope.P.getFilters(); + if (Filters.DumpModi) { + uint32_t Modi = *Filters.DumpModi; + SymbolGroup SG(&Input, Modi); + return iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(Modi)), + SG, Modi, Callback); + } + + uint32_t I = 0; + + for (const auto &SG : Input.symbol_groups()) { + if (shouldDumpSymbolGroup(I, SG, Filters)) + if (auto Err = + iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(I)), + SG, I, Callback)) + return Err; + + ++I; + } + return Error::success(); +} + +template <typename SubsectionT> +Error iterateModuleSubsections( + InputFile &File, const PrintScope &HeaderScope, + llvm::function_ref<Error(uint32_t, const SymbolGroup &, SubsectionT &)> + Callback) { + + return iterateSymbolGroups( + File, HeaderScope, [&](uint32_t Modi, const SymbolGroup &SG) -> Error { + for (const auto &SS : SG.getDebugSubsections()) { + SubsectionT Subsection; + + if (SS.kind() != Subsection.kind()) + continue; + + BinaryStreamReader Reader(SS.getRecordData()); + if (auto Err = Subsection.initialize(Reader)) + continue; + if (auto Err = Callback(Modi, SG, Subsection)) + return Err; + } + return Error::success(); + }); +} + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h b/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h new file mode 100644 index 000000000000..0db21309f593 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h @@ -0,0 +1,185 @@ +//===- LinePrinter.h ------------------------------------------ *- C++ --*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_LINEPRINTER_H +#define LLVM_DEBUGINFO_PDB_NATIVE_LINEPRINTER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/DebugInfo/PDB/Native/FormatUtil.h" +#include "llvm/Support/BinaryStreamRef.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/raw_ostream.h" + +#include <list> + +// Container for filter options to control which elements will be printed. +struct FilterOptions { + std::list<std::string> ExcludeTypes; + std::list<std::string> ExcludeSymbols; + std::list<std::string> ExcludeCompilands; + std::list<std::string> IncludeTypes; + std::list<std::string> IncludeSymbols; + std::list<std::string> IncludeCompilands; + uint32_t PaddingThreshold; + uint32_t SizeThreshold; + llvm::Optional<uint32_t> DumpModi; + llvm::Optional<uint32_t> ParentRecurseDepth; + llvm::Optional<uint32_t> ChildrenRecurseDepth; + llvm::Optional<uint32_t> SymbolOffset; + bool JustMyCode; +}; + +namespace llvm { +namespace msf { +class MSFStreamLayout; +} // namespace msf +namespace pdb { + +class ClassLayout; +class PDBFile; +class SymbolGroup; + +class LinePrinter { + friend class WithColor; + +public: + LinePrinter(int Indent, bool UseColor, raw_ostream &Stream, + const FilterOptions &Filters); + + void Indent(uint32_t Amount = 0); + void Unindent(uint32_t Amount = 0); + void NewLine(); + + void printLine(const Twine &T); + void print(const Twine &T); + template <typename... Ts> void formatLine(const char *Fmt, Ts &&...Items) { + printLine(formatv(Fmt, std::forward<Ts>(Items)...)); + } + template <typename... Ts> void format(const char *Fmt, Ts &&...Items) { + print(formatv(Fmt, std::forward<Ts>(Items)...)); + } + + void formatBinary(StringRef Label, ArrayRef<uint8_t> Data, + uint64_t StartOffset); + void formatBinary(StringRef Label, ArrayRef<uint8_t> Data, uint64_t BaseAddr, + uint64_t StartOffset); + + void formatMsfStreamData(StringRef Label, PDBFile &File, uint32_t StreamIdx, + StringRef StreamPurpose, uint64_t Offset, + uint64_t Size); + void formatMsfStreamData(StringRef Label, PDBFile &File, + const msf::MSFStreamLayout &Stream, + BinarySubstreamRef Substream); + void formatMsfStreamBlocks(PDBFile &File, const msf::MSFStreamLayout &Stream); + + bool hasColor() const { return UseColor; } + raw_ostream &getStream() { return OS; } + int getIndentLevel() const { return CurrentIndent; } + + bool IsClassExcluded(const ClassLayout &Class); + bool IsTypeExcluded(llvm::StringRef TypeName, uint64_t Size); + bool IsSymbolExcluded(llvm::StringRef SymbolName); + bool IsCompilandExcluded(llvm::StringRef CompilandName); + + const FilterOptions &getFilters() const { return Filters; } + +private: + template <typename Iter> + void SetFilters(std::list<Regex> &List, Iter Begin, Iter End) { + List.clear(); + for (; Begin != End; ++Begin) + List.emplace_back(StringRef(*Begin)); + } + + raw_ostream &OS; + int IndentSpaces; + int CurrentIndent; + bool UseColor; + const FilterOptions &Filters; + + std::list<Regex> ExcludeCompilandFilters; + std::list<Regex> ExcludeTypeFilters; + std::list<Regex> ExcludeSymbolFilters; + + std::list<Regex> IncludeCompilandFilters; + std::list<Regex> IncludeTypeFilters; + std::list<Regex> IncludeSymbolFilters; +}; + +struct PrintScope { + explicit PrintScope(LinePrinter &P, uint32_t IndentLevel) + : P(P), IndentLevel(IndentLevel) {} + explicit PrintScope(const PrintScope &Other, uint32_t LabelWidth) + : P(Other.P), IndentLevel(Other.IndentLevel), LabelWidth(LabelWidth) {} + + LinePrinter &P; + uint32_t IndentLevel; + uint32_t LabelWidth = 0; +}; + +inline PrintScope withLabelWidth(const PrintScope &Scope, uint32_t W) { + return PrintScope{Scope, W}; +} + +struct AutoIndent { + explicit AutoIndent(LinePrinter &L, uint32_t Amount = 0) + : L(&L), Amount(Amount) { + L.Indent(Amount); + } + explicit AutoIndent(const PrintScope &Scope) { + L = &Scope.P; + Amount = Scope.IndentLevel; + } + ~AutoIndent() { + if (L) + L->Unindent(Amount); + } + + LinePrinter *L = nullptr; + uint32_t Amount = 0; +}; + +template <class T> +inline raw_ostream &operator<<(LinePrinter &Printer, const T &Item) { + return Printer.getStream() << Item; +} + +enum class PDB_ColorItem { + None, + Address, + Type, + Comment, + Padding, + Keyword, + Offset, + Identifier, + Path, + SectionHeader, + LiteralValue, + Register, +}; + +class WithColor { +public: + WithColor(LinePrinter &P, PDB_ColorItem C); + ~WithColor(); + + raw_ostream &get() { return OS; } + +private: + void applyColor(PDB_ColorItem C); + raw_ostream &OS; + bool UseColor; +}; +} // namespace pdb +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h index cb1ffc729512..0caf9fffbad6 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h @@ -10,10 +10,8 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_MODULEDEBUGSTREAM_H #include "llvm/ADT/iterator_range.h" -#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" -#include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" @@ -21,10 +19,15 @@ #include <memory> namespace llvm { +class BinaryStreamReader; +namespace codeview { +class DebugChecksumsSubsectionRef; +} +namespace msf { +class MappedBlockStream; +} namespace pdb { -class DbiModuleDescriptor; - class ModuleDebugStreamRef { using DebugSubsectionIterator = codeview::DebugSubsectionArray::Iterator; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h index f110e90b3f90..18fbab0dd38c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h @@ -11,7 +11,6 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/PDB/Native/HashTable.h" #include "llvm/Support/Error.h" #include <cstdint> diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h index 073878afd129..c10e652efa8d 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h @@ -9,7 +9,7 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMGLOBALS_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMGLOBALS_H -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/PDBSymbol.h" diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumLineNumbers.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumLineNumbers.h index 32a4515d557e..a936b769d688 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumLineNumbers.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumLineNumbers.h @@ -9,16 +9,13 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMLINENUMBERS_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMLINENUMBERS_H -#include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" -#include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/IPDBLineNumber.h" #include "llvm/DebugInfo/PDB/Native/NativeLineNumber.h" +#include <vector> namespace llvm { namespace pdb { -class IPDBLineNumber; class NativeEnumLineNumbers : public IPDBEnumChildren<IPDBLineNumber> { public: diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbols.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbols.h index 480b3fb11419..5fc91675f209 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbols.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbols.h @@ -9,9 +9,9 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOLS_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOLS_H -#include "llvm/DebugInfo/CodeView/TypeRecord.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/PDBSymbol.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" #include <vector> diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h index 25c56567384f..2ca000c1c0fe 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h @@ -9,14 +9,17 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMTYPES_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMTYPES_H -#include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/PDBSymbol.h" #include <vector> namespace llvm { +namespace codeview { +class LazyRandomTypeCollection; +} namespace pdb { class NativeSession; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h index 280358d02305..82fdff130c4f 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h @@ -9,12 +9,15 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEEXESYMBOL_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEEXESYMBOL_H +#include "llvm/DebugInfo/CodeView/GUID.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { +class NativeSession; + class DbiStream; class NativeExeSymbol : public NativeRawSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h index b219055d2153..c15e22f61077 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h @@ -9,14 +9,17 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEFUNCTIONSYMBOL_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEFUNCTIONSYMBOL_H -#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { +class raw_ostream; namespace pdb { +class NativeSession; + class NativeFunctionSymbol : public NativeRawSymbol { public: NativeFunctionSymbol(NativeSession &Session, SymIndexId Id, diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeInlineSiteSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeInlineSiteSymbol.h index 2f6aba038ae8..3467ac912162 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeInlineSiteSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeInlineSiteSymbol.h @@ -9,14 +9,16 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEINLINESITESYMBOL_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEINLINESITESYMBOL_H -#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { +class NativeSession; + class NativeInlineSiteSymbol : public NativeRawSymbol { public: NativeInlineSiteSymbol(NativeSession &Session, SymIndexId Id, diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h index be0ddf0a063a..53f2985833fd 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h @@ -11,10 +11,12 @@ #include "llvm/DebugInfo/CodeView/Line.h" #include "llvm/DebugInfo/PDB/IPDBLineNumber.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" namespace llvm { namespace pdb { + +class NativeSession; + class NativeLineNumber : public IPDBLineNumber { public: explicit NativeLineNumber(const NativeSession &Session, diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h index 9f410e27f4cb..43de80507d02 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h @@ -9,13 +9,14 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEPUBLICSYMBOL_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEPUBLICSYMBOL_H -#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" namespace llvm { + +class raw_ostream; namespace pdb { +class NativeSession; class NativePublicSymbol : public NativeRawSymbol { public: diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h index 5f8fc587e546..95be7d09aae9 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h @@ -9,13 +9,11 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H -#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" -#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/SymbolCache.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" @@ -24,6 +22,12 @@ class MemoryBuffer; namespace pdb { class PDBFile; class NativeExeSymbol; +class IPDBSourceFile; +class ModuleDebugStreamRef; +class PDBSymbol; +class PDBSymbolCompiland; +class PDBSymbolExe; +template <typename ChildType> class IPDBEnumChildren; class NativeSession : public IPDBSession { struct PdbSearchOptions { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h index eb6336f268e8..c6653368bc0c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h @@ -11,11 +11,12 @@ #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" #include "llvm/DebugInfo/PDB/IPDBSourceFile.h" -#include "llvm/DebugInfo/PDB/Native/PDBFile.h" -#include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { +class PDBSymbolCompiland; +template <typename ChildType> class IPDBEnumChildren; class NativeSession; class NativeSourceFile : public IPDBSourceFile { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h index d6a3125ee40b..ab4abc4d3c2c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h @@ -9,12 +9,16 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVESYMBOLENUMERATOR_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVESYMBOLENUMERATOR_H -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { + +class raw_ostream; namespace pdb { +class NativeSession; class NativeTypeEnum; class NativeSymbolEnumerator : public NativeRawSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h index 2068c88fc74a..429c06f29ac7 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h @@ -10,12 +10,14 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEENUM_H #include "llvm/ADT/Optional.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { +class raw_ostream; namespace pdb { class NativeTypeBuiltin; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h index 90b5d8068959..47ea722313c3 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h @@ -9,17 +9,15 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEFUNCTIONSIG_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEFUNCTIONSIG_H -#include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { -class NativeTypeUDT; - class NativeTypeFunctionSig : public NativeRawSymbol { protected: void initialize() override; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h index 7a3dfaecefeb..1f357754ac0f 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h @@ -10,10 +10,11 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H #include "llvm/ADT/Optional.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h index 292fc48e7b6d..ce4ebcd00c4a 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h @@ -9,14 +9,19 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPETYPEDEF_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPETYPEDEF_H -#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { + +class raw_ostream; + namespace pdb { +class NativeSession; + class NativeTypeTypedef : public NativeRawSymbol { public: // Create a pointer record for a non-simple type. diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h index e1b31a256c12..a1dd39c0b4be 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h @@ -10,13 +10,17 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEUDT_H #include "llvm/ADT/Optional.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { + +class raw_ostream; namespace pdb { +class NativeSession; class NativeTypeUDT : public NativeRawSymbol { public: diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h index 21995ca665c1..92d51706c1da 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h @@ -9,13 +9,15 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEVTSHAPE_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEVTSHAPE_H -#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" namespace llvm { namespace pdb { +class NativeSession; class NativeTypeVTShape : public NativeRawSymbol { public: diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h index c5ee73280c46..1ea92ed4bf21 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h @@ -9,14 +9,12 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_PDBFILE_H #define LLVM_DEBUGINFO_PDB_NATIVE_PDBFILE_H -#include "llvm/ADT/DenseMap.h" #include "llvm/DebugInfo/MSF/IMSFFile.h" #include "llvm/DebugInfo/MSF/MSFCommon.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MathExtras.h" #include <memory> diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index 004d005280d4..c23d958f8ed0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -9,24 +9,28 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_PDBFILEBUILDER_H #define LLVM_DEBUGINFO_PDB_NATIVE_PDBFILEBUILDER_H -#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/PDB/Native/HashTable.h" #include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h" -#include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" -#include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include <memory> namespace llvm { +class WritableBinaryStream; +namespace codeview { +struct GUID; +} + namespace msf { class MSFBuilder; +struct MSFLayout; } namespace pdb { +struct SrcHeaderBlockEntry; class DbiStreamBuilder; class InfoStreamBuilder; class GSIStreamBuilder; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h index 5cb749c8a747..4336cd398baf 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h @@ -9,11 +9,9 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_PDBSTRINGTABLE_H #define LLVM_DEBUGINFO_PDB_NATIVE_PDBSTRINGTABLE_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" #include "llvm/Support/BinaryStreamArray.h" -#include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include <cstdint> @@ -21,10 +19,6 @@ namespace llvm { class BinaryStreamReader; -namespace msf { -class MappedBlockStream; -} - namespace pdb { struct PDBStringTableHeader; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h index bf6da3ea2920..a59a752ff911 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h @@ -9,20 +9,17 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_PUBLICSSTREAM_H #define LLVM_DEBUGINFO_PDB_NATIVE_PUBLICSSTREAM_H -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" -#include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" -#include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/Error.h" namespace llvm { +namespace msf { +class MappedBlockStream; +} namespace pdb { -class DbiStream; -struct GSIHashHeader; -class PDBFile; +struct PublicsStreamHeader; +struct SectionOffset; class PublicsStream { public: diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h index 1ff6ca173b2b..7c5b6b9e1bdf 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h @@ -10,23 +10,29 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLCACHE_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/IntervalMap.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/Line.h" -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/TypeDeserializer.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" -#include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeSourceFile.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" #include <memory> #include <vector> namespace llvm { +namespace codeview { +class InlineSiteSym; +struct FileChecksumEntry; +} // namespace codeview namespace pdb { +class IPDBSourceFile; +class NativeSession; +class PDBSymbol; +class PDBSymbolCompiland; class DbiStream; -class PDBFile; class SymbolCache { NativeSession &Session; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h index 839cc8d2c503..c2f7eb04d16e 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h @@ -9,7 +9,7 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLSTREAM_H #define LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLSTREAM_H -#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/Support/Error.h" @@ -18,7 +18,6 @@ namespace msf { class MappedBlockStream; } namespace pdb { -class PDBFile; class SymbolStream { public: @@ -41,7 +40,7 @@ private: codeview::CVSymbolArray SymbolRecords; std::unique_ptr<msf::MappedBlockStream> Stream; }; -} +} // namespace pdb } #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h index e49d58af4421..4c413abb2bf0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h @@ -12,22 +12,23 @@ #include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/PDB/Native/HashTable.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/BinaryStreamRef.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Error.h" namespace llvm { +class BinaryStream; namespace codeview { +class TypeIndex; +struct TypeIndexOffset; class LazyRandomTypeCollection; } namespace msf { class MappedBlockStream; } namespace pdb { +struct TpiStreamHeader; class PDBFile; class TpiStream { diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h index f18d38ae0b31..9f320358144c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h @@ -10,12 +10,10 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_TPISTREAMBUILDER_H #include "llvm/ADT/Optional.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/BinaryItemStream.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" @@ -23,7 +21,7 @@ namespace llvm { class BinaryByteStream; -class WritableBinaryStreamRef; +template <typename T> struct BinaryItemTraits; template <> struct BinaryItemTraits<llvm::codeview::CVType> { static size_t length(const codeview::CVType &Item) { return Item.length(); } @@ -32,16 +30,11 @@ template <> struct BinaryItemTraits<llvm::codeview::CVType> { } }; -namespace codeview { -class TypeRecord; -} namespace msf { class MSFBuilder; struct MSFLayout; } namespace pdb { -class PDBFile; -class TpiStream; struct TpiStreamHeader; class TpiStreamBuilder { @@ -88,7 +81,7 @@ private: const TpiStreamHeader *Header; uint32_t Idx; }; -} +} // namespace pdb } #endif diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBContext.h b/llvm/include/llvm/DebugInfo/PDB/PDBContext.h index 7b6793f0a639..3163c0a1dae0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBContext.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBContext.h @@ -45,6 +45,8 @@ namespace pdb { DILineInfo getLineInfoForAddress( object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; + DILineInfo + getLineInfoForDataAddress(object::SectionedAddress Address) override; DILineInfoTable getLineInfoForAddressRange( object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h index 24cf1e459f92..4e34b75b6117 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h @@ -9,11 +9,9 @@ #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOL_H #define LLVM_DEBUGINFO_PDB_PDBSYMBOL_H -#include "ConcreteSymbolEnumerator.h" #include "IPDBRawSymbol.h" #include "PDBExtras.h" #include "PDBTypes.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/Support/Casting.h" #define FORWARD_SYMBOL_METHOD(MethodName) \ @@ -43,6 +41,9 @@ class raw_ostream; namespace pdb { class IPDBSession; +class PDBSymDumper; +class PDBSymbol; +template <typename ChildType> class ConcreteSymbolEnumerator; #define DECLARE_PDB_SYMBOL_CONCRETE_TYPE(TagValue) \ private: \ diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h index c76466a97b66..c8d3d0b7bb96 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h @@ -13,7 +13,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolAnnotation : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h index cf471450d989..09142227b017 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h @@ -13,8 +13,6 @@ namespace llvm { -class raw_ostream; - namespace pdb { class PDBSymbolBlock : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h index dbd8ba5a63ff..46c159268533 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolCompilandDetails : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h index 61607a03593d..cba082f2ff19 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolCompilandEnv : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::CompilandEnv) diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h index 75a86411643a..c78b47ce9924 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h @@ -15,8 +15,6 @@ namespace llvm { -class raw_ostream; - namespace pdb { /// PDBSymbolCustom represents symbols that are compiler-specific and do not /// fit anywhere else in the lexical hierarchy. diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h index 7e9b69d7cf4b..61e67d1368a8 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h @@ -9,16 +9,16 @@ #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLDATA_H #define LLVM_DEBUGINFO_PDB_PDBSYMBOLDATA_H -#include "IPDBLineNumber.h" #include "PDBSymbol.h" #include "PDBTypes.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" namespace llvm { -class raw_ostream; - namespace pdb { +class PDBSymDumper; + class PDBSymbolData : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Data) public: diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h index f50057c68406..bfc7f7689718 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h @@ -9,17 +9,20 @@ #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNC_H #define LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNC_H -#include "IPDBLineNumber.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" + #include "PDBSymbol.h" -#include "PDBSymbolTypeFunctionSig.h" #include "PDBTypes.h" namespace llvm { -class raw_ostream; - namespace pdb { +class PDBSymDumper; +class PDBSymbolData; +class PDBSymbolTypeFunctionSig; +template <typename ChildType> class IPDBEnumChildren; + class PDBSymbolFunc : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Function) public: diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h index 1cdc1811bb1a..09c6f4728960 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h @@ -14,8 +14,6 @@ namespace llvm { -class raw_ostream; - namespace pdb { class PDBSymbolFuncDebugEnd : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h index 021f27c7f0f7..843a8348a2f0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolFuncDebugStart : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h index 33eb36696cc2..148802a47cbc 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolLabel : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h index f8dcb2ba9d5f..a757cc02624b 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolPublicSymbol : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h index a5f795cc1303..2b81a63995e6 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolThunk : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h index d4cd6e71423e..496141e5fa68 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeArray : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h index bd2dbc914725..c74ac3fb9cce 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h @@ -12,14 +12,14 @@ #include "PDBSymbol.h" #include "PDBTypes.h" -#include "llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h" -#include "llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" namespace llvm { -class raw_ostream; namespace pdb { +class PDBSymDumper; + class PDBSymbolTypeBaseClass : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BaseClass) public: diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index df6309b1545c..b923983095f3 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeBuiltin : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h index 7bf0317ff1ca..b15abf7bedfd 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeCustom : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h index 5d742237bac4..e7570b41dd21 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeDimension : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h index 0aab91039509..ee1f736c17a0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h @@ -9,16 +9,18 @@ #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEENUM_H #define LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEENUM_H -#include "IPDBLineNumber.h" #include "PDBSymbol.h" -#include "PDBSymbolTypeBuiltin.h" #include "PDBTypes.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" + namespace llvm { -class raw_ostream; namespace pdb { +class PDBSymDumper; +class PDBSymbolTypeBuiltin; + class PDBSymbolTypeEnum : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Enum) public: diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h index d56a90662dae..9fde42116261 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeFriend : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h index 559ceec5aace..71decff722a5 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeFunctionArg : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h index 5e7b83ce8004..866bf520a3b2 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeManaged : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h index da25eab50f9b..1b43ef9a21bd 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypePointer : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h index 8dc29ca26192..3f37730cf1df 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeTypedef : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h index 3e73ad7ac85a..a3a49a4b619a 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h @@ -9,18 +9,17 @@ #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEUDT_H #define LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEUDT_H -#include "IPDBLineNumber.h" -#include "IPDBSession.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" + #include "PDBSymbol.h" -#include "PDBSymbolTypeBaseClass.h" #include "PDBTypes.h" namespace llvm { -class raw_ostream; - namespace pdb { +class PDBSymDumper; + class PDBSymbolTypeUDT : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::UDT) public: diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h index d08728dafa76..6223bee98670 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeVTable : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h index c7e2ac148503..bec0a9970a9f 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolTypeVTableShape : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h index 5b4909b800b9..a53af49bc9e0 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h @@ -13,7 +13,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolUnknown : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h index 19a8f414eb43..dde25a023d00 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h @@ -14,7 +14,6 @@ namespace llvm { -class raw_ostream; namespace pdb { class PDBSymbolUsingNamespace : public PDBSymbol { diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h b/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h index e7c2ded1bee1..b6a794ad7e76 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h @@ -352,7 +352,8 @@ enum class PDB_BuiltinType { BSTR = 30, HResult = 31, Char16 = 32, - Char32 = 33 + Char32 = 33, + Char8 = 34, }; /// These values correspond to the flags that can be combined to control the diff --git a/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h b/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h index c67b093b63c0..8631c412f114 100644 --- a/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h +++ b/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h @@ -18,7 +18,6 @@ #include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h" #include "llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h" #include "llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h" -#include "llvm/DebugInfo/PDB/PDBTypes.h" #include <cstdint> #include <memory> #include <string> diff --git a/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h b/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h new file mode 100644 index 000000000000..c5340b5f0460 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h @@ -0,0 +1,51 @@ +//===-- llvm/DebugInfo/Symbolize/DIFetcher.h --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares a DIFetcher abstraction for obtaining debug info from an +/// arbitrary outside source. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H +#define LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H + +#include <cstdint> +#include <string> + +#include "llvm/ADT/ArrayRef.h" + +namespace llvm { +namespace symbolize { + +/// The DIFetcher interface provides arbitrary mechanisms for obtaining debug +/// info from an outside source. +class DIFetcher { +public: + virtual ~DIFetcher() = default; + virtual Optional<std::string> + fetchBuildID(ArrayRef<uint8_t> BuildID) const = 0; +}; + +/// LocalDIFetcher searches local cache directories for debug info. +class LocalDIFetcher : public DIFetcher { +public: + LocalDIFetcher(ArrayRef<std::string> DebugFileDirectory) + : DebugFileDirectory(DebugFileDirectory){}; + virtual ~LocalDIFetcher() = default; + + Optional<std::string> fetchBuildID(ArrayRef<uint8_t> BuildID) const override; + +private: + const ArrayRef<std::string> DebugFileDirectory; +}; + +} // end namespace symbolize +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Markup.h b/llvm/include/llvm/DebugInfo/Symbolize/Markup.h new file mode 100644 index 000000000000..2628b47cf6d3 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/Symbolize/Markup.h @@ -0,0 +1,120 @@ +//===- Markup.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares the log symbolizer markup data model and parser. +/// +/// See https://llvm.org/docs/SymbolizerMarkupFormat.html +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_MARKUP_H +#define LLVM_DEBUGINFO_SYMBOLIZE_MARKUP_H + +#include <iostream> + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Regex.h" + +namespace llvm { +namespace symbolize { + +/// A node of symbolizer markup. +/// +/// If only the Text field is set, this represents a region of text outside a +/// markup element. ANSI SGR control codes are also reported this way; if +/// detected, then the control code will be the entirety of the Text field, and +/// any surrounding text will be reported as preceding and following nodes. +struct MarkupNode { + /// The full text of this node in the input. + StringRef Text; + + /// If this represents an element, the tag. Otherwise, empty. + StringRef Tag; + + /// If this represents an element with fields, a list of the field contents. + /// Otherwise, empty. + SmallVector<StringRef> Fields; + + bool operator==(const MarkupNode &Other) const { + return Text == Other.Text && Tag == Other.Tag && Fields == Other.Fields; + } + bool operator!=(const MarkupNode &Other) const { return !(*this == Other); } +}; + +/// Parses a log containing symbolizer markup into a sequence of nodes. +class MarkupParser { +public: + MarkupParser(StringSet<> MultilineTags = {}); + + /// Parses an individual \p Line of input. + /// + /// Nodes from the previous parseLine() call that haven't yet been extracted + /// by nextNode() are discarded. The nodes returned by nextNode() may + /// reference the input string, so it must be retained by the caller until the + /// last use. + /// + /// Note that some elements may span multiple lines. If a line ends with the + /// start of one of these elements, then no nodes will be produced until the + /// either the end or something that cannot be part of an element is + /// encountered. This may only occur after multiple calls to parseLine(), + /// corresponding to the lines of the multi-line element. + void parseLine(StringRef Line); + + /// Inform the parser of that the input stream has ended. + /// + /// This allows the parser to finish any deferred processing (e.g., an + /// in-progress multi-line element) and may cause nextNode() to return + /// additional nodes. + void flush(); + + /// Returns the next node in the input sequence. + /// + /// Calling nextNode() may invalidate the contents of the node returned by the + /// previous call. + /// + /// \returns the next markup node or None if none remain. + Optional<MarkupNode> nextNode(); + +private: + Optional<MarkupNode> parseElement(StringRef Line); + void parseTextOutsideMarkup(StringRef Text); + Optional<StringRef> parseMultiLineBegin(StringRef Line); + Optional<StringRef> parseMultiLineEnd(StringRef Line); + + // Tags of elements that can span multiple lines. + const StringSet<> MultilineTags; + + // Contents of a multi-line element that has finished being parsed. Retained + // to keep returned StringRefs for the contents valid. + std::string FinishedMultiline; + + // Contents of a multi-line element that is still in the process of receiving + // lines. + std::string InProgressMultiline; + + // The line currently being parsed. + StringRef Line; + + // Buffer for nodes parsed from the current line. + SmallVector<MarkupNode> Buffer; + + // Next buffer index to return. + size_t NextIdx; + + // Regular expression matching supported ANSI SGR escape sequences. + const Regex SGRSyntax; +}; + +} // end namespace symbolize +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_MARKUP_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h b/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h new file mode 100644 index 000000000000..b7d70ccafe66 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h @@ -0,0 +1,76 @@ +//===- MarkupFilter.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares a filter that replaces symbolizer markup with +/// human-readable expressions. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_MARKUPFILTER_H +#define LLVM_DEBUGINFO_SYMBOLIZE_MARKUPFILTER_H + +#include "Markup.h" + +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace symbolize { + +/// Filter to convert parsed log symbolizer markup elements into human-readable +/// text. +class MarkupFilter { +public: + MarkupFilter(raw_ostream &OS, Optional<bool> ColorsEnabled = llvm::None); + + /// Begins a logical \p Line of markup. + /// + /// This must be called for each line of the input stream before calls to + /// filter() for elements of that line. The provided \p Line must be the same + /// one that was passed to parseLine() to produce the elements to be later + /// passed to filter(). + /// + /// This informs the filter that a new line is beginning and establishes a + /// context for error location reporting. + void beginLine(StringRef Line); + + /// Handle a \p Node of symbolizer markup. + /// + /// If the node is a recognized, valid markup element, it is replaced with a + /// human-readable string. If the node isn't an element or the element isn't + /// recognized, it is output verbatim. If the element is recognized but isn't + /// valid, it is omitted from the output. + void filter(const MarkupNode &Node); + +private: + bool trySGR(const MarkupNode &Node); + + void highlight(); + void restoreColor(); + void resetColor(); + + bool checkTag(const MarkupNode &Node) const; + bool checkNumFields(const MarkupNode &Node, size_t Size) const; + + void reportTypeError(StringRef Str, StringRef TypeName) const; + void reportLocation(StringRef::iterator Loc) const; + + raw_ostream &OS; + const bool ColorsEnabled; + + StringRef Line; + + Optional<raw_ostream::Colors> Color; + bool Bold = false; +}; + +} // end namespace symbolize +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_MARKUPFILTER_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/SymbolizableObjectFile.h b/llvm/include/llvm/DebugInfo/Symbolize/SymbolizableObjectFile.h new file mode 100644 index 000000000000..075dbe3e0e37 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/Symbolize/SymbolizableObjectFile.h @@ -0,0 +1,103 @@ +//===- SymbolizableObjectFile.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the SymbolizableObjectFile class. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEOBJECTFILE_H +#define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEOBJECTFILE_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" +#include "llvm/Support/Error.h" +#include <cstdint> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +namespace llvm { + +class DataExtractor; + +namespace symbolize { + +class SymbolizableObjectFile : public SymbolizableModule { +public: + static Expected<std::unique_ptr<SymbolizableObjectFile>> + create(const object::ObjectFile *Obj, std::unique_ptr<DIContext> DICtx, + bool UntagAddresses); + + DILineInfo symbolizeCode(object::SectionedAddress ModuleOffset, + DILineInfoSpecifier LineInfoSpecifier, + bool UseSymbolTable) const override; + DIInliningInfo symbolizeInlinedCode(object::SectionedAddress ModuleOffset, + DILineInfoSpecifier LineInfoSpecifier, + bool UseSymbolTable) const override; + DIGlobal symbolizeData(object::SectionedAddress ModuleOffset) const override; + std::vector<DILocal> + symbolizeFrame(object::SectionedAddress ModuleOffset) const override; + + // Return true if this is a 32-bit x86 PE COFF module. + bool isWin32Module() const override; + + // Returns the preferred base of the module, i.e. where the loader would place + // it in memory assuming there were no conflicts. + uint64_t getModulePreferredBase() const override; + +private: + bool shouldOverrideWithSymbolTable(FunctionNameKind FNKind, + bool UseSymbolTable) const; + + bool getNameFromSymbolTable(uint64_t Address, std::string &Name, + uint64_t &Addr, uint64_t &Size, + std::string &FileName) const; + // For big-endian PowerPC64 ELF, OpdAddress is the address of the .opd + // (function descriptor) section and OpdExtractor refers to its contents. + Error addSymbol(const object::SymbolRef &Symbol, uint64_t SymbolSize, + DataExtractor *OpdExtractor = nullptr, + uint64_t OpdAddress = 0); + Error addCoffExportSymbols(const object::COFFObjectFile *CoffObj); + + /// Search for the first occurence of specified Address in ObjectFile. + uint64_t getModuleSectionIndexForAddress(uint64_t Address) const; + + const object::ObjectFile *Module; + std::unique_ptr<DIContext> DebugInfoContext; + bool UntagAddresses; + + struct SymbolDesc { + uint64_t Addr; + // If size is 0, assume that symbol occupies the whole memory range up to + // the following symbol. + uint64_t Size; + + StringRef Name; + // Non-zero if this is an ELF local symbol. See the comment in + // getNameFromSymbolTable. + uint32_t ELFLocalSymIdx; + + bool operator<(const SymbolDesc &RHS) const { + return Addr != RHS.Addr ? Addr < RHS.Addr : Size < RHS.Size; + } + }; + std::vector<SymbolDesc> Symbols; + // (index, filename) pairs of ELF STT_FILE symbols. + std::vector<std::pair<uint32_t, StringRef>> FileSymbols; + + SymbolizableObjectFile(const object::ObjectFile *Obj, + std::unique_ptr<DIContext> DICtx, + bool UntagAddresses); +}; + +} // end namespace symbolize + +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEOBJECTFILE_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h index 4ec333422c4b..00c4bf0a615f 100644 --- a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -13,10 +13,12 @@ #ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H #define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H -#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/ilist_node.h" +#include "llvm/ADT/simple_ilist.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" #include "llvm/Object/Binary.h" -#include "llvm/Object/ELFObjectFile.h" -#include "llvm/Object/ObjectFile.h" #include "llvm/Support/Error.h" #include <algorithm> #include <cstdint> @@ -27,13 +29,24 @@ #include <vector> namespace llvm { +namespace object { +class ELFObjectFileBase; +class MachOObjectFile; +class ObjectFile; +struct SectionedAddress; +} // namespace object + namespace symbolize { +class SymbolizableModule; + using namespace object; using FunctionNameKind = DILineInfoSpecifier::FunctionNameKind; using FileLineInfoKind = DILineInfoSpecifier::FileLineInfoKind; +class CachedBinary; + class LLVMSymbolizer { public: struct Options { @@ -49,40 +62,63 @@ public: std::string FallbackDebugPath; std::string DWPName; std::vector<std::string> DebugFileDirectory; + size_t MaxCacheSize = + sizeof(size_t) == 4 + ? 512 * 1024 * 1024 /* 512 MiB */ + : static_cast<size_t>(4ULL * 1024 * 1024 * 1024) /* 4 GiB */; }; - LLVMSymbolizer() = default; - LLVMSymbolizer(const Options &Opts) : Opts(Opts) {} + LLVMSymbolizer(); + LLVMSymbolizer(const Options &Opts); - ~LLVMSymbolizer() { flush(); } + ~LLVMSymbolizer(); // Overloads accepting ObjectFile does not support COFF currently Expected<DILineInfo> symbolizeCode(const ObjectFile &Obj, object::SectionedAddress ModuleOffset); Expected<DILineInfo> symbolizeCode(const std::string &ModuleName, object::SectionedAddress ModuleOffset); + Expected<DILineInfo> symbolizeCode(ArrayRef<uint8_t> BuildID, + object::SectionedAddress ModuleOffset); Expected<DIInliningInfo> symbolizeInlinedCode(const ObjectFile &Obj, object::SectionedAddress ModuleOffset); Expected<DIInliningInfo> symbolizeInlinedCode(const std::string &ModuleName, object::SectionedAddress ModuleOffset); + Expected<DIInliningInfo> + symbolizeInlinedCode(ArrayRef<uint8_t> BuildID, + object::SectionedAddress ModuleOffset); Expected<DIGlobal> symbolizeData(const ObjectFile &Obj, object::SectionedAddress ModuleOffset); Expected<DIGlobal> symbolizeData(const std::string &ModuleName, object::SectionedAddress ModuleOffset); + Expected<DIGlobal> symbolizeData(ArrayRef<uint8_t> BuildID, + object::SectionedAddress ModuleOffset); Expected<std::vector<DILocal>> symbolizeFrame(const ObjectFile &Obj, object::SectionedAddress ModuleOffset); Expected<std::vector<DILocal>> symbolizeFrame(const std::string &ModuleName, object::SectionedAddress ModuleOffset); + Expected<std::vector<DILocal>> + symbolizeFrame(ArrayRef<uint8_t> BuildID, + object::SectionedAddress ModuleOffset); void flush(); + // Evict entries from the binary cache until it is under the maximum size + // given in the options. Calling this invalidates references in the DI... + // objects returned by the methods above. + void pruneCache(); + static std::string DemangleName(const std::string &Name, const SymbolizableModule *DbiModuleDescriptor); + void addDIFetcher(std::unique_ptr<DIFetcher> Fetcher) { + DIFetchers.push_back(std::move(Fetcher)); + } + private: // Bundles together object file with code/data and object file with // corresponding debug info. These objects can be the same. @@ -112,6 +148,12 @@ private: getOrCreateModuleInfo(const std::string &ModuleName); Expected<SymbolizableModule *> getOrCreateModuleInfo(const ObjectFile &Obj); + /// Returns a SymbolizableModule or an error if loading debug info failed. + /// Unlike the above, errors are reported each time, since they are more + /// likely to be transient. + Expected<SymbolizableModule *> + getOrCreateModuleInfo(ArrayRef<uint8_t> BuildID); + Expected<SymbolizableModule *> createModuleInfo(const ObjectFile *Obj, std::unique_ptr<DIContext> Context, StringRef ModuleName); @@ -126,6 +168,13 @@ private: const ELFObjectFileBase *Obj, const std::string &ArchName); + bool findDebugBinary(const std::string &OrigPath, + const std::string &DebuglinkName, uint32_t CRCHash, + std::string &Result); + + bool getOrFindDebugBinary(const ArrayRef<uint8_t> BuildID, + std::string &Result); + /// Returns pair of pointers to object and debug object. Expected<ObjectPair> getOrCreateObjectPair(const std::string &Path, const std::string &ArchName); @@ -136,15 +185,24 @@ private: Expected<ObjectFile *> getOrCreateObject(const std::string &Path, const std::string &ArchName); + /// Update the LRU cache order when a binary is accessed. + void recordAccess(CachedBinary &Bin); + std::map<std::string, std::unique_ptr<SymbolizableModule>, std::less<>> Modules; + StringMap<std::string> BuildIDPaths; /// Contains cached results of getOrCreateObjectPair(). std::map<std::pair<std::string, std::string>, ObjectPair> ObjectPairForPathArch; /// Contains parsed binary for each path, or parsing error. - std::map<std::string, OwningBinary<Binary>> BinaryForPath; + std::map<std::string, CachedBinary> BinaryForPath; + + /// A list of cached binaries in LRU order. + simple_ilist<CachedBinary> LRUBinaries; + /// Sum of the sizes of the cached binaries. + size_t CacheSize = 0; /// Parsed object file for path/architecture pair, where "path" refers /// to Mach-O universal binary. @@ -152,6 +210,37 @@ private: ObjectForUBPathAndArch; Options Opts; + + SmallVector<std::unique_ptr<DIFetcher>> DIFetchers; +}; + +// A binary intrusively linked into a LRU cache list. If the binary is empty, +// then the entry marks that an error occurred, and it is not part of the LRU +// list. +class CachedBinary : public ilist_node<CachedBinary> { +public: + CachedBinary() = default; + CachedBinary(OwningBinary<Binary> Bin) : Bin(std::move(Bin)) {} + + OwningBinary<Binary> &operator*() { return Bin; } + OwningBinary<Binary> *operator->() { return &Bin; } + + // Add an action to be performed when the binary is evicted, before all + // previously registered evictors. + void pushEvictor(std::function<void()> Evictor); + + // Run all registered evictors in the reverse of the order in which they were + // added. + void evict() { + if (Evictor) + Evictor(); + } + + size_t size() { return Bin.getBinary()->getData().size(); } + +private: + OwningBinary<Binary> Bin; + std::function<void()> Evictor; }; } // end namespace symbolize diff --git a/llvm/include/llvm/Debuginfod/DIFetcher.h b/llvm/include/llvm/Debuginfod/DIFetcher.h new file mode 100644 index 000000000000..d398fd900051 --- /dev/null +++ b/llvm/include/llvm/Debuginfod/DIFetcher.h @@ -0,0 +1,34 @@ +//===- llvm/DebugInfod/DIFetcher.h - Debug info fetcher----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares a DIFetcher implementation for obtaining debug info from +/// debuginfod. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFOD_DIFETCHER_H +#define LLVM_DEBUGINFOD_DIFETCHER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" + +namespace llvm { + +class DebuginfodDIFetcher : public symbolize::DIFetcher { +public: + virtual ~DebuginfodDIFetcher() = default; + + /// Fetches the given Build ID using debuginfod and returns a local path to + /// the resulting debug binary. + Optional<std::string> fetchBuildID(ArrayRef<uint8_t> BuildID) const override; +}; + +} // namespace llvm + +#endif // LLVM_DEBUGINFOD_DIFETCHER_H diff --git a/llvm/include/llvm/Debuginfod/HTTPClient.h b/llvm/include/llvm/Debuginfod/HTTPClient.h index ca3b76ca9f3f..6c94961032e7 100644 --- a/llvm/include/llvm/Debuginfod/HTTPClient.h +++ b/llvm/include/llvm/Debuginfod/HTTPClient.h @@ -7,9 +7,8 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This file contains the declarations of the HTTPClient, HTTPMethod, -/// HTTPResponseHandler, and BufferedHTTPResponseHandler classes, as well as -/// the HTTPResponseBuffer and HTTPRequest structs. +/// This file contains the declarations of the HTTPClient library for issuing +/// HTTP requests and handling the responses. /// //===----------------------------------------------------------------------===// @@ -40,43 +39,13 @@ bool operator==(const HTTPRequest &A, const HTTPRequest &B); /// of its methods. class HTTPResponseHandler { public: - /// Processes one line of HTTP response headers. - virtual Error handleHeaderLine(StringRef HeaderLine) = 0; - /// Processes an additional chunk of bytes of the HTTP response body. virtual Error handleBodyChunk(StringRef BodyChunk) = 0; - /// Processes the HTTP response status code. - virtual Error handleStatusCode(unsigned Code) = 0; - protected: ~HTTPResponseHandler(); }; -/// An HTTP response status code bundled with a buffer to store the body. -struct HTTPResponseBuffer { - unsigned Code = 0; - std::unique_ptr<WritableMemoryBuffer> Body; -}; - -/// A simple handler which writes returned data to an HTTPResponseBuffer. -/// Ignores all headers except the Content-Length, which it uses to -/// allocate an appropriately-sized Body buffer. -class BufferedHTTPResponseHandler final : public HTTPResponseHandler { - size_t Offset = 0; - -public: - /// Stores the data received from the HTTP server. - HTTPResponseBuffer ResponseBuffer; - - /// These callbacks store the body and status code in an HTTPResponseBuffer - /// allocated based on Content-Length. The Content-Length header must be - /// handled by handleHeaderLine before any calls to handleBodyChunk. - Error handleHeaderLine(StringRef HeaderLine) override; - Error handleBodyChunk(StringRef BodyChunk) override; - Error handleStatusCode(unsigned Code) override; -}; - /// A reusable client that can perform HTTPRequests through a network socket. class HTTPClient { #ifdef LLVM_ENABLE_CURL @@ -107,13 +76,8 @@ public: /// Handler method. Error perform(const HTTPRequest &Request, HTTPResponseHandler &Handler); - /// Performs the Request with the default BufferedHTTPResponseHandler, and - /// returns its HTTPResponseBuffer or an Error. - Expected<HTTPResponseBuffer> perform(const HTTPRequest &Request); - - /// Performs an HTTPRequest with the default configuration to make a GET - /// request to the given Url. Returns an HTTPResponseBuffer or an Error. - Expected<HTTPResponseBuffer> get(StringRef Url); + /// Returns the last received response code or zero if none. + unsigned responseCode(); }; } // end namespace llvm diff --git a/llvm/include/llvm/Demangle/Demangle.h b/llvm/include/llvm/Demangle/Demangle.h index 3150e049320b..6133d0b95bbf 100644 --- a/llvm/include/llvm/Demangle/Demangle.h +++ b/llvm/include/llvm/Demangle/Demangle.h @@ -57,8 +57,8 @@ char *microsoftDemangle(const char *mangled_name, size_t *n_read, char *buf, size_t *n_buf, int *status, MSDemangleFlags Flags = MSDF_None); -// Demangles a Rust v0 mangled symbol. The API follows that of __cxa_demangle. -char *rustDemangle(const char *MangledName, char *Buf, size_t *N, int *Status); +// Demangles a Rust v0 mangled symbol. +char *rustDemangle(const char *MangledName); // Demangles a D mangled symbol. char *dlangDemangle(const char *MangledName); diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h index 760319544a02..959632f13e1e 100644 --- a/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -16,10 +16,6 @@ #ifndef DEMANGLE_ITANIUMDEMANGLE_H #define DEMANGLE_ITANIUMDEMANGLE_H -// FIXME: (possibly) incomplete list of features that clang mangles that this -// file does not yet support: -// - C++ modules TS - #include "DemangleConfig.h" #include "StringView.h" #include "Utility.h" @@ -32,85 +28,6 @@ #include <limits> #include <utility> -#define FOR_EACH_NODE_KIND(X) \ - X(NodeArrayNode) \ - X(DotSuffix) \ - X(VendorExtQualType) \ - X(QualType) \ - X(ConversionOperatorType) \ - X(PostfixQualifiedType) \ - X(ElaboratedTypeSpefType) \ - X(NameType) \ - X(AbiTagAttr) \ - X(EnableIfAttr) \ - X(ObjCProtoName) \ - X(PointerType) \ - X(ReferenceType) \ - X(PointerToMemberType) \ - X(ArrayType) \ - X(FunctionType) \ - X(NoexceptSpec) \ - X(DynamicExceptionSpec) \ - X(FunctionEncoding) \ - X(LiteralOperator) \ - X(SpecialName) \ - X(CtorVtableSpecialName) \ - X(QualifiedName) \ - X(NestedName) \ - X(LocalName) \ - X(VectorType) \ - X(PixelVectorType) \ - X(BinaryFPType) \ - X(SyntheticTemplateParamName) \ - X(TypeTemplateParamDecl) \ - X(NonTypeTemplateParamDecl) \ - X(TemplateTemplateParamDecl) \ - X(TemplateParamPackDecl) \ - X(ParameterPack) \ - X(TemplateArgumentPack) \ - X(ParameterPackExpansion) \ - X(TemplateArgs) \ - X(ForwardTemplateReference) \ - X(NameWithTemplateArgs) \ - X(GlobalQualifiedName) \ - X(StdQualifiedName) \ - X(ExpandedSpecialSubstitution) \ - X(SpecialSubstitution) \ - X(CtorDtorName) \ - X(DtorName) \ - X(UnnamedTypeName) \ - X(ClosureTypeName) \ - X(StructuredBindingName) \ - X(BinaryExpr) \ - X(ArraySubscriptExpr) \ - X(PostfixExpr) \ - X(ConditionalExpr) \ - X(MemberExpr) \ - X(SubobjectExpr) \ - X(EnclosingExpr) \ - X(CastExpr) \ - X(SizeofParamPackExpr) \ - X(CallExpr) \ - X(NewExpr) \ - X(DeleteExpr) \ - X(PrefixExpr) \ - X(FunctionParam) \ - X(ConversionExpr) \ - X(PointerToMemberConversionExpr) \ - X(InitListExpr) \ - X(FoldExpr) \ - X(ThrowExpr) \ - X(BoolExpr) \ - X(StringLiteral) \ - X(LambdaExpr) \ - X(EnumLiteral) \ - X(IntegerLiteral) \ - X(FloatLiteral) \ - X(DoubleLiteral) \ - X(LongDoubleLiteral) \ - X(BracedExpr) \ - X(BracedRangeExpr) - DEMANGLE_NAMESPACE_BEGIN template <class T, size_t N> class PODSmallVector { @@ -238,37 +155,68 @@ public: class Node { public: enum Kind : unsigned char { -#define ENUMERATOR(NodeKind) K ## NodeKind, - FOR_EACH_NODE_KIND(ENUMERATOR) -#undef ENUMERATOR +#define NODE(NodeKind) K##NodeKind, +#include "ItaniumNodes.def" }; /// Three-way bool to track a cached value. Unknown is possible if this node /// has an unexpanded parameter pack below it that may affect this cache. enum class Cache : unsigned char { Yes, No, Unknown, }; + /// Operator precedence for expression nodes. Used to determine required + /// parens in expression emission. + enum class Prec { + Primary, + Postfix, + Unary, + Cast, + PtrMem, + Multiplicative, + Additive, + Shift, + Spaceship, + Relational, + Equality, + And, + Xor, + Ior, + AndIf, + OrIf, + Conditional, + Assign, + Comma, + Default, + }; + private: Kind K; + Prec Precedence : 6; + // FIXME: Make these protected. public: /// Tracks if this node has a component on its right side, in which case we /// need to call printRight. - Cache RHSComponentCache; + Cache RHSComponentCache : 2; /// Track if this node is a (possibly qualified) array type. This can affect /// how we format the output string. - Cache ArrayCache; + Cache ArrayCache : 2; /// Track if this node is a (possibly qualified) function type. This can /// affect how we format the output string. - Cache FunctionCache; + Cache FunctionCache : 2; public: - Node(Kind K_, Cache RHSComponentCache_ = Cache::No, - Cache ArrayCache_ = Cache::No, Cache FunctionCache_ = Cache::No) - : K(K_), RHSComponentCache(RHSComponentCache_), ArrayCache(ArrayCache_), - FunctionCache(FunctionCache_) {} + Node(Kind K_, Prec Precedence_ = Prec::Primary, + Cache RHSComponentCache_ = Cache::No, Cache ArrayCache_ = Cache::No, + Cache FunctionCache_ = Cache::No) + : K(K_), Precedence(Precedence_), RHSComponentCache(RHSComponentCache_), + ArrayCache(ArrayCache_), FunctionCache(FunctionCache_) {} + Node(Kind K_, Cache RHSComponentCache_, Cache ArrayCache_ = Cache::No, + Cache FunctionCache_ = Cache::No) + : Node(K_, Prec::Primary, RHSComponentCache_, ArrayCache_, + FunctionCache_) {} /// Visit the most-derived object corresponding to this object. template<typename Fn> void visit(Fn F) const; @@ -299,6 +247,8 @@ public: Kind getKind() const { return K; } + Prec getPrecedence() const { return Precedence; } + virtual bool hasRHSComponentSlow(OutputBuffer &) const { return false; } virtual bool hasArraySlow(OutputBuffer &) const { return false; } virtual bool hasFunctionSlow(OutputBuffer &) const { return false; } @@ -307,6 +257,19 @@ public: // get at a node that actually represents some concrete syntax. virtual const Node *getSyntaxNode(OutputBuffer &) const { return this; } + // Print this node as an expression operand, surrounding it in parentheses if + // its precedence is [Strictly] weaker than P. + void printAsOperand(OutputBuffer &OB, Prec P = Prec::Default, + bool StrictlyWorse = false) const { + bool Paren = + unsigned(getPrecedence()) >= unsigned(P) + unsigned(StrictlyWorse); + if (Paren) + OB.printOpen(); + print(OB); + if (Paren) + OB.printClose(); + } + void print(OutputBuffer &OB) const { printLeft(OB); if (RHSComponentCache != Cache::No) @@ -356,7 +319,7 @@ public: if (!FirstElement) OB += ", "; size_t AfterComma = OB.getCurrentPosition(); - Elements[Idx]->print(OB); + Elements[Idx]->printAsOperand(OB, Node::Prec::Comma); // Elements[Idx] is an empty parameter pack expansion, we should erase the // comma we just printed. @@ -494,7 +457,7 @@ class PostfixQualifiedType final : public Node { const StringView Postfix; public: - PostfixQualifiedType(Node *Ty_, StringView Postfix_) + PostfixQualifiedType(const Node *Ty_, StringView Postfix_) : Node(KPostfixQualifiedType), Ty(Ty_), Postfix(Postfix_) {} template<typename Fn> void match(Fn F) const { F(Ty, Postfix); } @@ -519,6 +482,26 @@ public: void printLeft(OutputBuffer &OB) const override { OB += Name; } }; +class BitIntType final : public Node { + const Node *Size; + bool Signed; + +public: + BitIntType(const Node *Size_, bool Signed_) + : Node(KBitIntType), Size(Size_), Signed(Signed_) {} + + template <typename Fn> void match(Fn F) const { F(Size, Signed); } + + void printLeft(OutputBuffer &OB) const override { + if (!Signed) + OB += "unsigned "; + OB += "_BitInt"; + OB.printOpen(); + Size->printAsOperand(OB); + OB.printClose(); + } +}; + class ElaboratedTypeSpefType : public Node { StringView Kind; Node *Child; @@ -693,7 +676,7 @@ public: void printLeft(OutputBuffer &OB) const override { if (Printing) return; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); std::pair<ReferenceKind, const Node *> Collapsed = collapse(OB); if (!Collapsed.second) return; @@ -708,7 +691,7 @@ public: void printRight(OutputBuffer &OB) const override { if (Printing) return; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); std::pair<ReferenceKind, const Node *> Collapsed = collapse(OB); if (!Collapsed.second) return; @@ -815,9 +798,9 @@ public: } void printRight(OutputBuffer &OB) const override { - OB += "("; + OB.printOpen(); Params.printWithComma(OB); - OB += ")"; + OB.printClose(); Ret->printRight(OB); if (CVQuals & QualConst) @@ -847,9 +830,10 @@ public: template<typename Fn> void match(Fn F) const { F(E); } void printLeft(OutputBuffer &OB) const override { - OB += "noexcept("; - E->print(OB); - OB += ")"; + OB += "noexcept"; + OB.printOpen(); + E->printAsOperand(OB); + OB.printClose(); } }; @@ -862,9 +846,10 @@ public: template<typename Fn> void match(Fn F) const { F(Types); } void printLeft(OutputBuffer &OB) const override { - OB += "throw("; + OB += "throw"; + OB.printOpen(); Types.printWithComma(OB); - OB += ')'; + OB.printClose(); } }; @@ -910,9 +895,9 @@ public: } void printRight(OutputBuffer &OB) const override { - OB += "("; + OB.printOpen(); Params.printWithComma(OB); - OB += ")"; + OB.printClose(); if (Ret) Ret->printRight(OB); @@ -1001,6 +986,46 @@ struct NestedName : Node { } }; +struct ModuleName : Node { + ModuleName *Parent; + Node *Name; + bool IsPartition; + + ModuleName(ModuleName *Parent_, Node *Name_, bool IsPartition_ = false) + : Node(KModuleName), Parent(Parent_), Name(Name_), + IsPartition(IsPartition_) {} + + template <typename Fn> void match(Fn F) const { + F(Parent, Name, IsPartition); + } + + void printLeft(OutputBuffer &OB) const override { + if (Parent) + Parent->print(OB); + if (Parent || IsPartition) + OB += IsPartition ? ':' : '.'; + Name->print(OB); + } +}; + +struct ModuleEntity : Node { + ModuleName *Module; + Node *Name; + + ModuleEntity(ModuleName *Module_, Node *Name_) + : Node(KModuleEntity), Module(Module_), Name(Name_) {} + + template <typename Fn> void match(Fn F) const { F(Module, Name); } + + StringView getBaseName() const override { return Name->getBaseName(); } + + void printLeft(OutputBuffer &OB) const override { + Name->print(OB); + OB += '@'; + Module->print(OB); + } +}; + struct LocalName : Node { Node *Encoding; Node *Entity; @@ -1042,9 +1067,8 @@ class VectorType final : public Node { const Node *Dimension; public: - VectorType(const Node *BaseType_, Node *Dimension_) - : Node(KVectorType), BaseType(BaseType_), - Dimension(Dimension_) {} + VectorType(const Node *BaseType_, const Node *Dimension_) + : Node(KVectorType), BaseType(BaseType_), Dimension(Dimension_) {} template<typename Fn> void match(Fn F) const { F(BaseType, Dimension); } @@ -1176,6 +1200,7 @@ public: template<typename Fn> void match(Fn F) const { F(Name, Params); } void printLeft(OutputBuffer &OB) const override { + ScopedOverride<unsigned> LT(OB.GtIsGt, 0); OB += "template<"; Params.printWithComma(OB); OB += "> typename "; @@ -1311,8 +1336,8 @@ public: void printLeft(OutputBuffer &OB) const override { constexpr unsigned Max = std::numeric_limits<unsigned>::max(); - SwapAndRestore<unsigned> SavePackIdx(OB.CurrentPackIndex, Max); - SwapAndRestore<unsigned> SavePackMax(OB.CurrentPackMax, Max); + ScopedOverride<unsigned> SavePackIdx(OB.CurrentPackIndex, Max); + ScopedOverride<unsigned> SavePackMax(OB.CurrentPackMax, Max); size_t StreamPos = OB.getCurrentPosition(); // Print the first element in the pack. If Child contains a ParameterPack, @@ -1353,10 +1378,9 @@ public: NodeArray getParams() { return Params; } void printLeft(OutputBuffer &OB) const override { + ScopedOverride<unsigned> LT(OB.GtIsGt, 0); OB += "<"; Params.printWithComma(OB); - if (OB.back() == '>') - OB += " "; OB += ">"; } }; @@ -1402,38 +1426,38 @@ struct ForwardTemplateReference : Node { bool hasRHSComponentSlow(OutputBuffer &OB) const override { if (Printing) return false; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); return Ref->hasRHSComponent(OB); } bool hasArraySlow(OutputBuffer &OB) const override { if (Printing) return false; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); return Ref->hasArray(OB); } bool hasFunctionSlow(OutputBuffer &OB) const override { if (Printing) return false; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); return Ref->hasFunction(OB); } const Node *getSyntaxNode(OutputBuffer &OB) const override { if (Printing) return this; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); return Ref->getSyntaxNode(OB); } void printLeft(OutputBuffer &OB) const override { if (Printing) return; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); Ref->printLeft(OB); } void printRight(OutputBuffer &OB) const override { if (Printing) return; - SwapAndRestore<bool> SavePrinting(Printing, true); + ScopedOverride<bool> SavePrinting(Printing, true); Ref->printRight(OB); } }; @@ -1473,21 +1497,6 @@ public: } }; -struct StdQualifiedName : Node { - Node *Child; - - StdQualifiedName(Node *Child_) : Node(KStdQualifiedName), Child(Child_) {} - - template<typename Fn> void match(Fn F) const { F(Child); } - - StringView getBaseName() const override { return Child->getBaseName(); } - - void printLeft(OutputBuffer &OB) const override { - OB += "std::"; - Child->print(OB); - } -}; - enum class SpecialSubKind { allocator, basic_string, @@ -1497,15 +1506,25 @@ enum class SpecialSubKind { iostream, }; -class ExpandedSpecialSubstitution final : public Node { +class SpecialSubstitution; +class ExpandedSpecialSubstitution : public Node { +protected: SpecialSubKind SSK; + ExpandedSpecialSubstitution(SpecialSubKind SSK_, Kind K_) + : Node(K_), SSK(SSK_) {} public: ExpandedSpecialSubstitution(SpecialSubKind SSK_) - : Node(KExpandedSpecialSubstitution), SSK(SSK_) {} + : ExpandedSpecialSubstitution(SSK_, KExpandedSpecialSubstitution) {} + inline ExpandedSpecialSubstitution(SpecialSubstitution const *); template<typename Fn> void match(Fn F) const { F(SSK); } +protected: + bool isInstantiation() const { + return unsigned(SSK) >= unsigned(SpecialSubKind::string); + } + StringView getBaseName() const override { switch (SSK) { case SpecialSubKind::allocator: @@ -1524,82 +1543,44 @@ public: DEMANGLE_UNREACHABLE; } +private: void printLeft(OutputBuffer &OB) const override { - switch (SSK) { - case SpecialSubKind::allocator: - OB += "std::allocator"; - break; - case SpecialSubKind::basic_string: - OB += "std::basic_string"; - break; - case SpecialSubKind::string: - OB += "std::basic_string<char, std::char_traits<char>, " - "std::allocator<char> >"; - break; - case SpecialSubKind::istream: - OB += "std::basic_istream<char, std::char_traits<char> >"; - break; - case SpecialSubKind::ostream: - OB += "std::basic_ostream<char, std::char_traits<char> >"; - break; - case SpecialSubKind::iostream: - OB += "std::basic_iostream<char, std::char_traits<char> >"; - break; + OB << "std::" << getBaseName(); + if (isInstantiation()) { + OB << "<char, std::char_traits<char>"; + if (SSK == SpecialSubKind::string) + OB << ", std::allocator<char>"; + OB << ">"; } } }; -class SpecialSubstitution final : public Node { +class SpecialSubstitution final : public ExpandedSpecialSubstitution { public: - SpecialSubKind SSK; - SpecialSubstitution(SpecialSubKind SSK_) - : Node(KSpecialSubstitution), SSK(SSK_) {} + : ExpandedSpecialSubstitution(SSK_, KSpecialSubstitution) {} template<typename Fn> void match(Fn F) const { F(SSK); } StringView getBaseName() const override { - switch (SSK) { - case SpecialSubKind::allocator: - return StringView("allocator"); - case SpecialSubKind::basic_string: - return StringView("basic_string"); - case SpecialSubKind::string: - return StringView("string"); - case SpecialSubKind::istream: - return StringView("istream"); - case SpecialSubKind::ostream: - return StringView("ostream"); - case SpecialSubKind::iostream: - return StringView("iostream"); + auto SV = ExpandedSpecialSubstitution::getBaseName (); + if (isInstantiation()) { + // The instantiations are typedefs that drop the "basic_" prefix. + assert(SV.startsWith("basic_")); + SV = SV.dropFront(sizeof("basic_") - 1); } - DEMANGLE_UNREACHABLE; + return SV; } void printLeft(OutputBuffer &OB) const override { - switch (SSK) { - case SpecialSubKind::allocator: - OB += "std::allocator"; - break; - case SpecialSubKind::basic_string: - OB += "std::basic_string"; - break; - case SpecialSubKind::string: - OB += "std::string"; - break; - case SpecialSubKind::istream: - OB += "std::istream"; - break; - case SpecialSubKind::ostream: - OB += "std::ostream"; - break; - case SpecialSubKind::iostream: - OB += "std::iostream"; - break; - } + OB << "std::" << getBaseName(); } }; +inline ExpandedSpecialSubstitution::ExpandedSpecialSubstitution( + SpecialSubstitution const *SS) + : ExpandedSpecialSubstitution(SS->SSK) {} + class CtorDtorName final : public Node { const Node *Basename; const bool IsDtor; @@ -1665,13 +1646,14 @@ public: void printDeclarator(OutputBuffer &OB) const { if (!TemplateParams.empty()) { + ScopedOverride<unsigned> LT(OB.GtIsGt, 0); OB += "<"; TemplateParams.printWithComma(OB); OB += ">"; } - OB += "("; + OB.printOpen(); Params.printWithComma(OB); - OB += ")"; + OB.printClose(); } void printLeft(OutputBuffer &OB) const override { @@ -1691,9 +1673,9 @@ public: template<typename Fn> void match(Fn F) const { F(Bindings); } void printLeft(OutputBuffer &OB) const override { - OB += '['; + OB.printOpen('['); Bindings.printWithComma(OB); - OB += ']'; + OB.printClose(']'); } }; @@ -1705,28 +1687,31 @@ class BinaryExpr : public Node { const Node *RHS; public: - BinaryExpr(const Node *LHS_, StringView InfixOperator_, const Node *RHS_) - : Node(KBinaryExpr), LHS(LHS_), InfixOperator(InfixOperator_), RHS(RHS_) { - } + BinaryExpr(const Node *LHS_, StringView InfixOperator_, const Node *RHS_, + Prec Prec_) + : Node(KBinaryExpr, Prec_), LHS(LHS_), InfixOperator(InfixOperator_), + RHS(RHS_) {} - template<typename Fn> void match(Fn F) const { F(LHS, InfixOperator, RHS); } + template <typename Fn> void match(Fn F) const { + F(LHS, InfixOperator, RHS, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - // might be a template argument expression, then we need to disambiguate - // with parens. - if (InfixOperator == ">") - OB += "("; - - OB += "("; - LHS->print(OB); - OB += ") "; + bool ParenAll = OB.isGtInsideTemplateArgs() && + (InfixOperator == ">" || InfixOperator == ">>"); + if (ParenAll) + OB.printOpen(); + // Assignment is right associative, with special LHS precedence. + bool IsAssign = getPrecedence() == Prec::Assign; + LHS->printAsOperand(OB, IsAssign ? Prec::OrIf : getPrecedence(), !IsAssign); + // No space before comma operator + if (!(InfixOperator == ",")) + OB += " "; OB += InfixOperator; - OB += " ("; - RHS->print(OB); - OB += ")"; - - if (InfixOperator == ">") - OB += ")"; + OB += " "; + RHS->printAsOperand(OB, getPrecedence(), IsAssign); + if (ParenAll) + OB.printClose(); } }; @@ -1735,17 +1720,18 @@ class ArraySubscriptExpr : public Node { const Node *Op2; public: - ArraySubscriptExpr(const Node *Op1_, const Node *Op2_) - : Node(KArraySubscriptExpr), Op1(Op1_), Op2(Op2_) {} + ArraySubscriptExpr(const Node *Op1_, const Node *Op2_, Prec Prec_) + : Node(KArraySubscriptExpr, Prec_), Op1(Op1_), Op2(Op2_) {} - template<typename Fn> void match(Fn F) const { F(Op1, Op2); } + template <typename Fn> void match(Fn F) const { + F(Op1, Op2, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - OB += "("; - Op1->print(OB); - OB += ")["; - Op2->print(OB); - OB += "]"; + Op1->printAsOperand(OB, getPrecedence()); + OB.printOpen('['); + Op2->printAsOperand(OB); + OB.printClose(']'); } }; @@ -1754,15 +1740,15 @@ class PostfixExpr : public Node { const StringView Operator; public: - PostfixExpr(const Node *Child_, StringView Operator_) - : Node(KPostfixExpr), Child(Child_), Operator(Operator_) {} + PostfixExpr(const Node *Child_, StringView Operator_, Prec Prec_) + : Node(KPostfixExpr, Prec_), Child(Child_), Operator(Operator_) {} - template<typename Fn> void match(Fn F) const { F(Child, Operator); } + template <typename Fn> void match(Fn F) const { + F(Child, Operator, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - OB += "("; - Child->print(OB); - OB += ")"; + Child->printAsOperand(OB, getPrecedence(), true); OB += Operator; } }; @@ -1773,19 +1759,20 @@ class ConditionalExpr : public Node { const Node *Else; public: - ConditionalExpr(const Node *Cond_, const Node *Then_, const Node *Else_) - : Node(KConditionalExpr), Cond(Cond_), Then(Then_), Else(Else_) {} + ConditionalExpr(const Node *Cond_, const Node *Then_, const Node *Else_, + Prec Prec_) + : Node(KConditionalExpr, Prec_), Cond(Cond_), Then(Then_), Else(Else_) {} - template<typename Fn> void match(Fn F) const { F(Cond, Then, Else); } + template <typename Fn> void match(Fn F) const { + F(Cond, Then, Else, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - OB += "("; - Cond->print(OB); - OB += ") ? ("; - Then->print(OB); - OB += ") : ("; - Else->print(OB); - OB += ")"; + Cond->printAsOperand(OB, getPrecedence()); + OB += " ? "; + Then->printAsOperand(OB); + OB += " : "; + Else->printAsOperand(OB, Prec::Assign, true); } }; @@ -1795,15 +1782,17 @@ class MemberExpr : public Node { const Node *RHS; public: - MemberExpr(const Node *LHS_, StringView Kind_, const Node *RHS_) - : Node(KMemberExpr), LHS(LHS_), Kind(Kind_), RHS(RHS_) {} + MemberExpr(const Node *LHS_, StringView Kind_, const Node *RHS_, Prec Prec_) + : Node(KMemberExpr, Prec_), LHS(LHS_), Kind(Kind_), RHS(RHS_) {} - template<typename Fn> void match(Fn F) const { F(LHS, Kind, RHS); } + template <typename Fn> void match(Fn F) const { + F(LHS, Kind, RHS, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - LHS->print(OB); + LHS->printAsOperand(OB, getPrecedence(), true); OB += Kind; - RHS->print(OB); + RHS->printAsOperand(OB, getPrecedence(), false); } }; @@ -1847,15 +1836,19 @@ class EnclosingExpr : public Node { const StringView Postfix; public: - EnclosingExpr(StringView Prefix_, Node *Infix_, StringView Postfix_) - : Node(KEnclosingExpr), Prefix(Prefix_), Infix(Infix_), - Postfix(Postfix_) {} + EnclosingExpr(StringView Prefix_, const Node *Infix_, + Prec Prec_ = Prec::Primary) + : Node(KEnclosingExpr, Prec_), Prefix(Prefix_), Infix(Infix_) {} - template<typename Fn> void match(Fn F) const { F(Prefix, Infix, Postfix); } + template <typename Fn> void match(Fn F) const { + F(Prefix, Infix, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { OB += Prefix; + OB.printOpen(); Infix->print(OB); + OB.printClose(); OB += Postfix; } }; @@ -1867,18 +1860,24 @@ class CastExpr : public Node { const Node *From; public: - CastExpr(StringView CastKind_, const Node *To_, const Node *From_) - : Node(KCastExpr), CastKind(CastKind_), To(To_), From(From_) {} + CastExpr(StringView CastKind_, const Node *To_, const Node *From_, Prec Prec_) + : Node(KCastExpr, Prec_), CastKind(CastKind_), To(To_), From(From_) {} - template<typename Fn> void match(Fn F) const { F(CastKind, To, From); } + template <typename Fn> void match(Fn F) const { + F(CastKind, To, From, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { OB += CastKind; - OB += "<"; - To->printLeft(OB); - OB += ">("; - From->printLeft(OB); - OB += ")"; + { + ScopedOverride<unsigned> LT(OB.GtIsGt, 0); + OB += "<"; + To->printLeft(OB); + OB += ">"; + } + OB.printOpen(); + From->printAsOperand(OB); + OB.printClose(); } }; @@ -1892,10 +1891,11 @@ public: template<typename Fn> void match(Fn F) const { F(Pack); } void printLeft(OutputBuffer &OB) const override { - OB += "sizeof...("; + OB += "sizeof..."; + OB.printOpen(); ParameterPackExpansion PPE(Pack); PPE.printLeft(OB); - OB += ")"; + OB.printClose(); } }; @@ -1904,16 +1904,18 @@ class CallExpr : public Node { NodeArray Args; public: - CallExpr(const Node *Callee_, NodeArray Args_) - : Node(KCallExpr), Callee(Callee_), Args(Args_) {} + CallExpr(const Node *Callee_, NodeArray Args_, Prec Prec_) + : Node(KCallExpr, Prec_), Callee(Callee_), Args(Args_) {} - template<typename Fn> void match(Fn F) const { F(Callee, Args); } + template <typename Fn> void match(Fn F) const { + F(Callee, Args, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { Callee->print(OB); - OB += "("; + OB.printOpen(); Args.printWithComma(OB); - OB += ")"; + OB.printClose(); } }; @@ -1926,31 +1928,31 @@ class NewExpr : public Node { bool IsArray; // new[] ? public: NewExpr(NodeArray ExprList_, Node *Type_, NodeArray InitList_, bool IsGlobal_, - bool IsArray_) - : Node(KNewExpr), ExprList(ExprList_), Type(Type_), InitList(InitList_), - IsGlobal(IsGlobal_), IsArray(IsArray_) {} + bool IsArray_, Prec Prec_) + : Node(KNewExpr, Prec_), ExprList(ExprList_), Type(Type_), + InitList(InitList_), IsGlobal(IsGlobal_), IsArray(IsArray_) {} template<typename Fn> void match(Fn F) const { - F(ExprList, Type, InitList, IsGlobal, IsArray); + F(ExprList, Type, InitList, IsGlobal, IsArray, getPrecedence()); } void printLeft(OutputBuffer &OB) const override { if (IsGlobal) - OB += "::operator "; + OB += "::"; OB += "new"; if (IsArray) OB += "[]"; - OB += ' '; if (!ExprList.empty()) { - OB += "("; + OB.printOpen(); ExprList.printWithComma(OB); - OB += ")"; + OB.printClose(); } + OB += " "; Type->print(OB); if (!InitList.empty()) { - OB += "("; + OB.printOpen(); InitList.printWithComma(OB); - OB += ")"; + OB.printClose(); } } }; @@ -1961,17 +1963,21 @@ class DeleteExpr : public Node { bool IsArray; public: - DeleteExpr(Node *Op_, bool IsGlobal_, bool IsArray_) - : Node(KDeleteExpr), Op(Op_), IsGlobal(IsGlobal_), IsArray(IsArray_) {} + DeleteExpr(Node *Op_, bool IsGlobal_, bool IsArray_, Prec Prec_) + : Node(KDeleteExpr, Prec_), Op(Op_), IsGlobal(IsGlobal_), + IsArray(IsArray_) {} - template<typename Fn> void match(Fn F) const { F(Op, IsGlobal, IsArray); } + template <typename Fn> void match(Fn F) const { + F(Op, IsGlobal, IsArray, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { if (IsGlobal) OB += "::"; OB += "delete"; if (IsArray) - OB += "[] "; + OB += "[]"; + OB += ' '; Op->print(OB); } }; @@ -1981,16 +1987,16 @@ class PrefixExpr : public Node { Node *Child; public: - PrefixExpr(StringView Prefix_, Node *Child_) - : Node(KPrefixExpr), Prefix(Prefix_), Child(Child_) {} + PrefixExpr(StringView Prefix_, Node *Child_, Prec Prec_) + : Node(KPrefixExpr, Prec_), Prefix(Prefix_), Child(Child_) {} - template<typename Fn> void match(Fn F) const { F(Prefix, Child); } + template <typename Fn> void match(Fn F) const { + F(Prefix, Child, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { OB += Prefix; - OB += "("; - Child->print(OB); - OB += ")"; + Child->printAsOperand(OB, getPrecedence()); } }; @@ -2013,17 +2019,20 @@ class ConversionExpr : public Node { NodeArray Expressions; public: - ConversionExpr(const Node *Type_, NodeArray Expressions_) - : Node(KConversionExpr), Type(Type_), Expressions(Expressions_) {} + ConversionExpr(const Node *Type_, NodeArray Expressions_, Prec Prec_) + : Node(KConversionExpr, Prec_), Type(Type_), Expressions(Expressions_) {} - template<typename Fn> void match(Fn F) const { F(Type, Expressions); } + template <typename Fn> void match(Fn F) const { + F(Type, Expressions, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - OB += "("; + OB.printOpen(); Type->print(OB); - OB += ")("; + OB.printClose(); + OB.printOpen(); Expressions.printWithComma(OB); - OB += ")"; + OB.printClose(); } }; @@ -2034,18 +2043,21 @@ class PointerToMemberConversionExpr : public Node { public: PointerToMemberConversionExpr(const Node *Type_, const Node *SubExpr_, - StringView Offset_) - : Node(KPointerToMemberConversionExpr), Type(Type_), SubExpr(SubExpr_), - Offset(Offset_) {} + StringView Offset_, Prec Prec_) + : Node(KPointerToMemberConversionExpr, Prec_), Type(Type_), + SubExpr(SubExpr_), Offset(Offset_) {} - template<typename Fn> void match(Fn F) const { F(Type, SubExpr, Offset); } + template <typename Fn> void match(Fn F) const { + F(Type, SubExpr, Offset, getPrecedence()); + } void printLeft(OutputBuffer &OB) const override { - OB += "("; + OB.printOpen(); Type->print(OB); - OB += ")("; + OB.printClose(); + OB.printOpen(); SubExpr->print(OB); - OB += ")"; + OB.printClose(); } }; @@ -2131,41 +2143,33 @@ public: void printLeft(OutputBuffer &OB) const override { auto PrintPack = [&] { - OB += '('; + OB.printOpen(); ParameterPackExpansion(Pack).print(OB); - OB += ')'; + OB.printClose(); }; - OB += '('; - - if (IsLeftFold) { - // init op ... op pack - if (Init != nullptr) { - Init->print(OB); - OB += ' '; - OB += OperatorName; - OB += ' '; - } - // ... op pack - OB += "... "; - OB += OperatorName; - OB += ' '; - PrintPack(); - } else { // !IsLeftFold - // pack op ... - PrintPack(); - OB += ' '; - OB += OperatorName; - OB += " ..."; - // pack op ... op init - if (Init != nullptr) { - OB += ' '; - OB += OperatorName; - OB += ' '; - Init->print(OB); - } + OB.printOpen(); + // Either '[init op ]... op pack' or 'pack op ...[ op init]' + // Refactored to '[(init|pack) op ]...[ op (pack|init)]' + // Fold expr operands are cast-expressions + if (!IsLeftFold || Init != nullptr) { + // '(init|pack) op ' + if (IsLeftFold) + Init->printAsOperand(OB, Prec::Cast, true); + else + PrintPack(); + OB << " " << OperatorName << " "; + } + OB << "..."; + if (IsLeftFold || Init != nullptr) { + // ' op (init|pack)' + OB << " " << OperatorName << " "; + if (IsLeftFold) + PrintPack(); + else + Init->printAsOperand(OB, Prec::Cast, true); } - OB += ')'; + OB.printClose(); } }; @@ -2239,9 +2243,9 @@ public: template<typename Fn> void match(Fn F) const { F(Ty, Integer); } void printLeft(OutputBuffer &OB) const override { - OB << "("; + OB.printOpen(); Ty->print(OB); - OB << ")"; + OB.printClose(); if (Integer[0] == 'n') OB << "-" << Integer.dropFront(1); @@ -2262,13 +2266,13 @@ public: void printLeft(OutputBuffer &OB) const override { if (Type.size() > 3) { - OB += "("; + OB.printOpen(); OB += Type; - OB += ")"; + OB.printClose(); } if (Value[0] == 'n') { - OB += "-"; + OB += '-'; OB += Value.dropFront(1); } else OB += Value; @@ -2344,24 +2348,22 @@ using LongDoubleLiteral = FloatLiteralImpl<long double>; template<typename Fn> void Node::visit(Fn F) const { switch (K) { -#define CASE(X) case K ## X: return F(static_cast<const X*>(this)); - FOR_EACH_NODE_KIND(CASE) -#undef CASE +#define NODE(X) \ + case K##X: \ + return F(static_cast<const X *>(this)); +#include "ItaniumNodes.def" } assert(0 && "unknown mangling node kind"); } /// Determine the kind of a node from its type. template<typename NodeT> struct NodeKind; -#define SPECIALIZATION(X) \ - template<> struct NodeKind<X> { \ - static constexpr Node::Kind Kind = Node::K##X; \ - static constexpr const char *name() { return #X; } \ +#define NODE(X) \ + template <> struct NodeKind<X> { \ + static constexpr Node::Kind Kind = Node::K##X; \ + static constexpr const char *name() { return #X; } \ }; -FOR_EACH_NODE_KIND(SPECIALIZATION) -#undef SPECIALIZATION - -#undef FOR_EACH_NODE_KIND +#include "ItaniumNodes.def" template <typename Derived, typename Alloc> struct AbstractManglingParser { const char *First; @@ -2499,17 +2501,16 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { /// Parse the <expr> production. Node *parseExpr(); - Node *parsePrefixExpr(StringView Kind); - Node *parseBinaryExpr(StringView Kind); + Node *parsePrefixExpr(StringView Kind, Node::Prec Prec); + Node *parseBinaryExpr(StringView Kind, Node::Prec Prec); Node *parseIntegerLiteral(StringView Lit); Node *parseExprPrimary(); template <class Float> Node *parseFloatingLiteral(); Node *parseFunctionParam(); - Node *parseNewExpr(); Node *parseConversionExpr(); Node *parseBracedExpr(); Node *parseFoldExpr(); - Node *parsePointerToMemberConversionExpr(); + Node *parsePointerToMemberConversionExpr(Node::Prec Prec); Node *parseSubobjectExpr(); /// Parse the <type> production. @@ -2557,17 +2558,80 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Node *parseName(NameState *State = nullptr); Node *parseLocalName(NameState *State); Node *parseOperatorName(NameState *State); - Node *parseUnqualifiedName(NameState *State); + bool parseModuleNameOpt(ModuleName *&Module); + Node *parseUnqualifiedName(NameState *State, Node *Scope, ModuleName *Module); Node *parseUnnamedTypeName(NameState *State); Node *parseSourceName(NameState *State); - Node *parseUnscopedName(NameState *State); + Node *parseUnscopedName(NameState *State, bool *isSubstName); Node *parseNestedName(NameState *State); Node *parseCtorDtorName(Node *&SoFar, NameState *State); Node *parseAbiTags(Node *N); + struct OperatorInfo { + enum OIKind : unsigned char { + Prefix, // Prefix unary: @ expr + Postfix, // Postfix unary: expr @ + Binary, // Binary: lhs @ rhs + Array, // Array index: lhs [ rhs ] + Member, // Member access: lhs @ rhs + New, // New + Del, // Delete + Call, // Function call: expr (expr*) + CCast, // C cast: (type)expr + Conditional, // Conditional: expr ? expr : expr + NameOnly, // Overload only, not allowed in expression. + // Below do not have operator names + NamedCast, // Named cast, @<type>(expr) + OfIdOp, // alignof, sizeof, typeid + + Unnameable = NamedCast, + }; + char Enc[2]; // Encoding + OIKind Kind; // Kind of operator + bool Flag : 1; // Entry-specific flag + Node::Prec Prec : 7; // Precedence + const char *Name; // Spelling + + public: + constexpr OperatorInfo(const char (&E)[3], OIKind K, bool F, Node::Prec P, + const char *N) + : Enc{E[0], E[1]}, Kind{K}, Flag{F}, Prec{P}, Name{N} {} + + public: + bool operator<(const OperatorInfo &Other) const { + return *this < Other.Enc; + } + bool operator<(const char *Peek) const { + return Enc[0] < Peek[0] || (Enc[0] == Peek[0] && Enc[1] < Peek[1]); + } + bool operator==(const char *Peek) const { + return Enc[0] == Peek[0] && Enc[1] == Peek[1]; + } + bool operator!=(const char *Peek) const { return !this->operator==(Peek); } + + public: + StringView getSymbol() const { + StringView Res = Name; + if (Kind < Unnameable) { + assert(Res.startsWith("operator") && + "operator name does not start with 'operator'"); + Res = Res.dropFront(sizeof("operator") - 1); + Res.consumeFront(' '); + } + return Res; + } + StringView getName() const { return Name; } + OIKind getKind() const { return Kind; } + bool getFlag() const { return Flag; } + Node::Prec getPrecedence() const { return Prec; } + }; + static const OperatorInfo Ops[]; + static const size_t NumOps; + const OperatorInfo *parseOperatorEncoding(); + /// Parse the <unresolved-name> production. - Node *parseUnresolvedName(); + Node *parseUnresolvedName(bool Global); Node *parseSimpleId(); Node *parseBaseUnresolvedName(); Node *parseUnresolvedType(); @@ -2588,26 +2652,16 @@ const char* parse_discriminator(const char* first, const char* last); // ::= <substitution> template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseName(NameState *State) { - consumeIf('L'); // extension - if (look() == 'N') return getDerived().parseNestedName(State); if (look() == 'Z') return getDerived().parseLocalName(State); Node *Result = nullptr; - bool IsSubst = look() == 'S' && look(1) != 't'; - if (IsSubst) { - // A substitution must lead to: - // ::= <unscoped-template-name> <template-args> - Result = getDerived().parseSubstitution(); - } else { - // An unscoped name can be one of: - // ::= <unscoped-name> - // ::= <unscoped-template-name> <template-args> - Result = getDerived().parseUnscopedName(State); - } - if (Result == nullptr) + bool IsSubst = false; + + Result = getDerived().parseUnscopedName(State, &IsSubst); + if (!Result) return nullptr; if (look() == 'I') { @@ -2667,38 +2721,63 @@ Node *AbstractManglingParser<Derived, Alloc>::parseLocalName(NameState *State) { // <unscoped-name> ::= <unqualified-name> // ::= St <unqualified-name> # ::std:: -// extension ::= StL<unqualified-name> +// [*] extension template <typename Derived, typename Alloc> Node * -AbstractManglingParser<Derived, Alloc>::parseUnscopedName(NameState *State) { - bool IsStd = consumeIf("St"); - if (IsStd) - consumeIf('L'); +AbstractManglingParser<Derived, Alloc>::parseUnscopedName(NameState *State, + bool *IsSubst) { - Node *Result = getDerived().parseUnqualifiedName(State); - if (Result == nullptr) - return nullptr; - if (IsStd) - Result = make<StdQualifiedName>(Result); + Node *Std = nullptr; + if (consumeIf("St")) { + Std = make<NameType>("std"); + if (Std == nullptr) + return nullptr; + } - return Result; + Node *Res = nullptr; + ModuleName *Module = nullptr; + if (look() == 'S') { + Node *S = getDerived().parseSubstitution(); + if (!S) + return nullptr; + if (S->getKind() == Node::KModuleName) + Module = static_cast<ModuleName *>(S); + else if (IsSubst && Std == nullptr) { + Res = S; + *IsSubst = true; + } else { + return nullptr; + } + } + + if (Res == nullptr || Std != nullptr) { + Res = getDerived().parseUnqualifiedName(State, Std, Module); + } + + return Res; } -// <unqualified-name> ::= <operator-name> [abi-tags] -// ::= <ctor-dtor-name> -// ::= <source-name> -// ::= <unnamed-type-name> -// ::= DC <source-name>+ E # structured binding declaration +// <unqualified-name> ::= [<module-name>] L? <operator-name> [<abi-tags>] +// ::= [<module-name>] <ctor-dtor-name> [<abi-tags>] +// ::= [<module-name>] L? <source-name> [<abi-tags>] +// ::= [<module-name>] L? <unnamed-type-name> [<abi-tags>] +// # structured binding declaration +// ::= [<module-name>] L? DC <source-name>+ E template <typename Derived, typename Alloc> -Node * -AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName(NameState *State) { - // <ctor-dtor-name>s are special-cased in parseNestedName(). +Node *AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName( + NameState *State, Node *Scope, ModuleName *Module) { + if (getDerived().parseModuleNameOpt(Module)) + return nullptr; + + consumeIf('L'); + Node *Result; - if (look() == 'U') - Result = getDerived().parseUnnamedTypeName(State); - else if (look() >= '1' && look() <= '9') + if (look() >= '1' && look() <= '9') { Result = getDerived().parseSourceName(State); - else if (consumeIf("DC")) { + } else if (look() == 'U') { + Result = getDerived().parseUnnamedTypeName(State); + } else if (consumeIf("DC")) { + // Structured binding size_t BindingsBegin = Names.size(); do { Node *Binding = getDerived().parseSourceName(State); @@ -2707,13 +2786,46 @@ AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName(NameState *State) { Names.push_back(Binding); } while (!consumeIf('E')); Result = make<StructuredBindingName>(popTrailingNodeArray(BindingsBegin)); - } else + } else if (look() == 'C' || look() == 'D') { + // A <ctor-dtor-name>. + if (Scope == nullptr || Module != nullptr) + return nullptr; + Result = getDerived().parseCtorDtorName(Scope, State); + } else { Result = getDerived().parseOperatorName(State); + } + + if (Result != nullptr && Module != nullptr) + Result = make<ModuleEntity>(Module, Result); if (Result != nullptr) Result = getDerived().parseAbiTags(Result); + if (Result != nullptr && Scope != nullptr) + Result = make<NestedName>(Scope, Result); + return Result; } +// <module-name> ::= <module-subname> +// ::= <module-name> <module-subname> +// ::= <substitution> # passed in by caller +// <module-subname> ::= W <source-name> +// ::= W P <source-name> +template <typename Derived, typename Alloc> +bool AbstractManglingParser<Derived, Alloc>::parseModuleNameOpt( + ModuleName *&Module) { + while (consumeIf('W')) { + bool IsPartition = consumeIf('P'); + Node *Sub = getDerived().parseSourceName(nullptr); + if (!Sub) + return true; + Module = + static_cast<ModuleName *>(make<ModuleName>(Module, Sub, IsPartition)); + Subs.push_back(Module); + } + + return false; +} + // <unnamed-type-name> ::= Ut [<nonnegative number>] _ // ::= <closure-type-name> // @@ -2735,7 +2847,7 @@ AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *State) { return make<UnnamedTypeName>(Count); } if (consumeIf("Ul")) { - SwapAndRestore<size_t> SwapParams(ParsingLambdaParamsAtLevel, + ScopedOverride<size_t> SwapParams(ParsingLambdaParamsAtLevel, TemplateParams.size()); ScopedTemplateParamList LambdaTemplateParams(this); @@ -2813,97 +2925,124 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSourceName(NameState *) { return make<NameType>(Name); } -// <operator-name> ::= aa # && -// ::= ad # & (unary) -// ::= an # & -// ::= aN # &= -// ::= aS # = -// ::= cl # () -// ::= cm # , -// ::= co # ~ -// ::= cv <type> # (cast) -// ::= da # delete[] -// ::= de # * (unary) -// ::= dl # delete -// ::= dv # / -// ::= dV # /= -// ::= eo # ^ -// ::= eO # ^= -// ::= eq # == -// ::= ge # >= -// ::= gt # > -// ::= ix # [] -// ::= le # <= +// Operator encodings +template <typename Derived, typename Alloc> +const typename AbstractManglingParser< + Derived, Alloc>::OperatorInfo AbstractManglingParser<Derived, + Alloc>::Ops[] = { + // Keep ordered by encoding + {"aN", OperatorInfo::Binary, false, Node::Prec::Assign, "operator&="}, + {"aS", OperatorInfo::Binary, false, Node::Prec::Assign, "operator="}, + {"aa", OperatorInfo::Binary, false, Node::Prec::AndIf, "operator&&"}, + {"ad", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator&"}, + {"an", OperatorInfo::Binary, false, Node::Prec::And, "operator&"}, + {"at", OperatorInfo::OfIdOp, /*Type*/ true, Node::Prec::Unary, "alignof "}, + {"aw", OperatorInfo::NameOnly, false, Node::Prec::Primary, + "operator co_await"}, + {"az", OperatorInfo::OfIdOp, /*Type*/ false, Node::Prec::Unary, "alignof "}, + {"cc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, "const_cast"}, + {"cl", OperatorInfo::Call, false, Node::Prec::Postfix, "operator()"}, + {"cm", OperatorInfo::Binary, false, Node::Prec::Comma, "operator,"}, + {"co", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator~"}, + {"cv", OperatorInfo::CCast, false, Node::Prec::Cast, "operator"}, // C Cast + {"dV", OperatorInfo::Binary, false, Node::Prec::Assign, "operator/="}, + {"da", OperatorInfo::Del, /*Ary*/ true, Node::Prec::Unary, + "operator delete[]"}, + {"dc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, "dynamic_cast"}, + {"de", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator*"}, + {"dl", OperatorInfo::Del, /*Ary*/ false, Node::Prec::Unary, + "operator delete"}, + {"ds", OperatorInfo::Member, /*Named*/ false, Node::Prec::PtrMem, + "operator.*"}, + {"dt", OperatorInfo::Member, /*Named*/ false, Node::Prec::Postfix, + "operator."}, + {"dv", OperatorInfo::Binary, false, Node::Prec::Assign, "operator/"}, + {"eO", OperatorInfo::Binary, false, Node::Prec::Assign, "operator^="}, + {"eo", OperatorInfo::Binary, false, Node::Prec::Xor, "operator^"}, + {"eq", OperatorInfo::Binary, false, Node::Prec::Equality, "operator=="}, + {"ge", OperatorInfo::Binary, false, Node::Prec::Relational, "operator>="}, + {"gt", OperatorInfo::Binary, false, Node::Prec::Relational, "operator>"}, + {"ix", OperatorInfo::Array, false, Node::Prec::Postfix, "operator[]"}, + {"lS", OperatorInfo::Binary, false, Node::Prec::Assign, "operator<<="}, + {"le", OperatorInfo::Binary, false, Node::Prec::Relational, "operator<="}, + {"ls", OperatorInfo::Binary, false, Node::Prec::Shift, "operator<<"}, + {"lt", OperatorInfo::Binary, false, Node::Prec::Relational, "operator<"}, + {"mI", OperatorInfo::Binary, false, Node::Prec::Assign, "operator-="}, + {"mL", OperatorInfo::Binary, false, Node::Prec::Assign, "operator*="}, + {"mi", OperatorInfo::Binary, false, Node::Prec::Additive, "operator-"}, + {"ml", OperatorInfo::Binary, false, Node::Prec::Multiplicative, + "operator*"}, + {"mm", OperatorInfo::Postfix, false, Node::Prec::Postfix, "operator--"}, + {"na", OperatorInfo::New, /*Ary*/ true, Node::Prec::Unary, + "operator new[]"}, + {"ne", OperatorInfo::Binary, false, Node::Prec::Equality, "operator!="}, + {"ng", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator-"}, + {"nt", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator!"}, + {"nw", OperatorInfo::New, /*Ary*/ false, Node::Prec::Unary, "operator new"}, + {"oR", OperatorInfo::Binary, false, Node::Prec::Assign, "operator|="}, + {"oo", OperatorInfo::Binary, false, Node::Prec::OrIf, "operator||"}, + {"or", OperatorInfo::Binary, false, Node::Prec::Ior, "operator|"}, + {"pL", OperatorInfo::Binary, false, Node::Prec::Assign, "operator+="}, + {"pl", OperatorInfo::Binary, false, Node::Prec::Additive, "operator+"}, + {"pm", OperatorInfo::Member, /*Named*/ false, Node::Prec::PtrMem, + "operator->*"}, + {"pp", OperatorInfo::Postfix, false, Node::Prec::Postfix, "operator++"}, + {"ps", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator+"}, + {"pt", OperatorInfo::Member, /*Named*/ true, Node::Prec::Postfix, + "operator->"}, + {"qu", OperatorInfo::Conditional, false, Node::Prec::Conditional, + "operator?"}, + {"rM", OperatorInfo::Binary, false, Node::Prec::Assign, "operator%="}, + {"rS", OperatorInfo::Binary, false, Node::Prec::Assign, "operator>>="}, + {"rc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, + "reinterpret_cast"}, + {"rm", OperatorInfo::Binary, false, Node::Prec::Multiplicative, + "operator%"}, + {"rs", OperatorInfo::Binary, false, Node::Prec::Shift, "operator>>"}, + {"sc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, "static_cast"}, + {"ss", OperatorInfo::Binary, false, Node::Prec::Spaceship, "operator<=>"}, + {"st", OperatorInfo::OfIdOp, /*Type*/ true, Node::Prec::Unary, "sizeof "}, + {"sz", OperatorInfo::OfIdOp, /*Type*/ false, Node::Prec::Unary, "sizeof "}, + {"te", OperatorInfo::OfIdOp, /*Type*/ false, Node::Prec::Postfix, + "typeid "}, + {"ti", OperatorInfo::OfIdOp, /*Type*/ true, Node::Prec::Postfix, "typeid "}, +}; +template <typename Derived, typename Alloc> +const size_t AbstractManglingParser<Derived, Alloc>::NumOps = sizeof(Ops) / + sizeof(Ops[0]); + +// If the next 2 chars are an operator encoding, consume them and return their +// OperatorInfo. Otherwise return nullptr. +template <typename Derived, typename Alloc> +const typename AbstractManglingParser<Derived, Alloc>::OperatorInfo * +AbstractManglingParser<Derived, Alloc>::parseOperatorEncoding() { + if (numLeft() < 2) + return nullptr; + + auto Op = std::lower_bound( + &Ops[0], &Ops[NumOps], First, + [](const OperatorInfo &Op_, const char *Enc_) { return Op_ < Enc_; }); + if (Op == &Ops[NumOps] || *Op != First) + return nullptr; + + First += 2; + return Op; +} + +// <operator-name> ::= See parseOperatorEncoding() // ::= li <source-name> # operator "" -// ::= ls # << -// ::= lS # <<= -// ::= lt # < -// ::= mi # - -// ::= mI # -= -// ::= ml # * -// ::= mL # *= -// ::= mm # -- (postfix in <expression> context) -// ::= na # new[] -// ::= ne # != -// ::= ng # - (unary) -// ::= nt # ! -// ::= nw # new -// ::= oo # || -// ::= or # | -// ::= oR # |= -// ::= pm # ->* -// ::= pl # + -// ::= pL # += -// ::= pp # ++ (postfix in <expression> context) -// ::= ps # + (unary) -// ::= pt # -> -// ::= qu # ? -// ::= rm # % -// ::= rM # %= -// ::= rs # >> -// ::= rS # >>= -// ::= ss # <=> C++2a -// ::= v <digit> <source-name> # vendor extended operator +// ::= v <digit> <source-name> # vendor extended operator template <typename Derived, typename Alloc> Node * AbstractManglingParser<Derived, Alloc>::parseOperatorName(NameState *State) { - switch (look()) { - case 'a': - switch (look(1)) { - case 'a': - First += 2; - return make<NameType>("operator&&"); - case 'd': - case 'n': - First += 2; - return make<NameType>("operator&"); - case 'N': - First += 2; - return make<NameType>("operator&="); - case 'S': - First += 2; - return make<NameType>("operator="); - } - return nullptr; - case 'c': - switch (look(1)) { - case 'l': - First += 2; - return make<NameType>("operator()"); - case 'm': - First += 2; - return make<NameType>("operator,"); - case 'o': - First += 2; - return make<NameType>("operator~"); - // ::= cv <type> # (cast) - case 'v': { - First += 2; - SwapAndRestore<bool> SaveTemplate(TryToParseTemplateArgs, false); + if (const auto *Op = parseOperatorEncoding()) { + if (Op->getKind() == OperatorInfo::CCast) { + // ::= cv <type> # (cast) + ScopedOverride<bool> SaveTemplate(TryToParseTemplateArgs, false); // If we're parsing an encoding, State != nullptr and the conversion // operators' <type> could have a <template-param> that refers to some // <template-arg>s further ahead in the mangled name. - SwapAndRestore<bool> SavePermit(PermitForwardTemplateReferences, + ScopedOverride<bool> SavePermit(PermitForwardTemplateReferences, PermitForwardTemplateReferences || State != nullptr); Node *Ty = getDerived().parseType(); @@ -2912,185 +3051,29 @@ AbstractManglingParser<Derived, Alloc>::parseOperatorName(NameState *State) { if (State) State->CtorDtorConversion = true; return make<ConversionOperatorType>(Ty); } - } - return nullptr; - case 'd': - switch (look(1)) { - case 'a': - First += 2; - return make<NameType>("operator delete[]"); - case 'e': - First += 2; - return make<NameType>("operator*"); - case 'l': - First += 2; - return make<NameType>("operator delete"); - case 'v': - First += 2; - return make<NameType>("operator/"); - case 'V': - First += 2; - return make<NameType>("operator/="); - } - return nullptr; - case 'e': - switch (look(1)) { - case 'o': - First += 2; - return make<NameType>("operator^"); - case 'O': - First += 2; - return make<NameType>("operator^="); - case 'q': - First += 2; - return make<NameType>("operator=="); - } - return nullptr; - case 'g': - switch (look(1)) { - case 'e': - First += 2; - return make<NameType>("operator>="); - case 't': - First += 2; - return make<NameType>("operator>"); - } - return nullptr; - case 'i': - if (look(1) == 'x') { - First += 2; - return make<NameType>("operator[]"); - } - return nullptr; - case 'l': - switch (look(1)) { - case 'e': - First += 2; - return make<NameType>("operator<="); + + if (Op->getKind() >= OperatorInfo::Unnameable) + /* Not a nameable operator. */ + return nullptr; + if (Op->getKind() == OperatorInfo::Member && !Op->getFlag()) + /* Not a nameable MemberExpr */ + return nullptr; + + return make<NameType>(Op->getName()); + } + + if (consumeIf("li")) { // ::= li <source-name> # operator "" - case 'i': { - First += 2; - Node *SN = getDerived().parseSourceName(State); - if (SN == nullptr) - return nullptr; - return make<LiteralOperator>(SN); - } - case 's': - First += 2; - return make<NameType>("operator<<"); - case 'S': - First += 2; - return make<NameType>("operator<<="); - case 't': - First += 2; - return make<NameType>("operator<"); - } - return nullptr; - case 'm': - switch (look(1)) { - case 'i': - First += 2; - return make<NameType>("operator-"); - case 'I': - First += 2; - return make<NameType>("operator-="); - case 'l': - First += 2; - return make<NameType>("operator*"); - case 'L': - First += 2; - return make<NameType>("operator*="); - case 'm': - First += 2; - return make<NameType>("operator--"); - } - return nullptr; - case 'n': - switch (look(1)) { - case 'a': - First += 2; - return make<NameType>("operator new[]"); - case 'e': - First += 2; - return make<NameType>("operator!="); - case 'g': - First += 2; - return make<NameType>("operator-"); - case 't': - First += 2; - return make<NameType>("operator!"); - case 'w': - First += 2; - return make<NameType>("operator new"); - } - return nullptr; - case 'o': - switch (look(1)) { - case 'o': - First += 2; - return make<NameType>("operator||"); - case 'r': - First += 2; - return make<NameType>("operator|"); - case 'R': - First += 2; - return make<NameType>("operator|="); - } - return nullptr; - case 'p': - switch (look(1)) { - case 'm': - First += 2; - return make<NameType>("operator->*"); - case 'l': - First += 2; - return make<NameType>("operator+"); - case 'L': - First += 2; - return make<NameType>("operator+="); - case 'p': - First += 2; - return make<NameType>("operator++"); - case 's': - First += 2; - return make<NameType>("operator+"); - case 't': - First += 2; - return make<NameType>("operator->"); - } - return nullptr; - case 'q': - if (look(1) == 'u') { - First += 2; - return make<NameType>("operator?"); - } - return nullptr; - case 'r': - switch (look(1)) { - case 'm': - First += 2; - return make<NameType>("operator%"); - case 'M': - First += 2; - return make<NameType>("operator%="); - case 's': - First += 2; - return make<NameType>("operator>>"); - case 'S': - First += 2; - return make<NameType>("operator>>="); - } - return nullptr; - case 's': - if (look(1) == 's') { - First += 2; - return make<NameType>("operator<=>"); - } - return nullptr; - // ::= v <digit> <source-name> # vendor extended operator - case 'v': - if (std::isdigit(look(1))) { - First += 2; + Node *SN = getDerived().parseSourceName(State); + if (SN == nullptr) + return nullptr; + return make<LiteralOperator>(SN); + } + + if (consumeIf('v')) { + // ::= v <digit> <source-name> # vendor extended operator + if (look() >= '0' && look() <= '9') { + First++; Node *SN = getDerived().parseSourceName(State); if (SN == nullptr) return nullptr; @@ -3098,6 +3081,7 @@ AbstractManglingParser<Derived, Alloc>::parseOperatorName(NameState *State) { } return nullptr; } + return nullptr; } @@ -3116,19 +3100,11 @@ Node * AbstractManglingParser<Derived, Alloc>::parseCtorDtorName(Node *&SoFar, NameState *State) { if (SoFar->getKind() == Node::KSpecialSubstitution) { - auto SSK = static_cast<SpecialSubstitution *>(SoFar)->SSK; - switch (SSK) { - case SpecialSubKind::string: - case SpecialSubKind::istream: - case SpecialSubKind::ostream: - case SpecialSubKind::iostream: - SoFar = make<ExpandedSpecialSubstitution>(SSK); - if (!SoFar) - return nullptr; - break; - default: - break; - } + // Expand the special substitution. + SoFar = make<ExpandedSpecialSubstitution>( + static_cast<SpecialSubstitution *>(SoFar)); + if (!SoFar) + return nullptr; } if (consumeIf('C')) { @@ -3157,8 +3133,10 @@ AbstractManglingParser<Derived, Alloc>::parseCtorDtorName(Node *&SoFar, return nullptr; } -// <nested-name> ::= N [<CV-Qualifiers>] [<ref-qualifier>] <prefix> <unqualified-name> E -// ::= N [<CV-Qualifiers>] [<ref-qualifier>] <template-prefix> <template-args> E +// <nested-name> ::= N [<CV-Qualifiers>] [<ref-qualifier>] <prefix> +// <unqualified-name> E +// ::= N [<CV-Qualifiers>] [<ref-qualifier>] <template-prefix> +// <template-args> E // // <prefix> ::= <prefix> <unqualified-name> // ::= <template-prefix> <template-args> @@ -3167,7 +3145,7 @@ AbstractManglingParser<Derived, Alloc>::parseCtorDtorName(Node *&SoFar, // ::= # empty // ::= <substitution> // ::= <prefix> <data-member-prefix> -// extension ::= L +// [*] extension // // <data-member-prefix> := <member source-name> [<template-args>] M // @@ -3187,90 +3165,76 @@ AbstractManglingParser<Derived, Alloc>::parseNestedName(NameState *State) { if (State) State->ReferenceQualifier = FrefQualRValue; } else if (consumeIf('R')) { if (State) State->ReferenceQualifier = FrefQualLValue; - } else + } else { if (State) State->ReferenceQualifier = FrefQualNone; - - Node *SoFar = nullptr; - auto PushComponent = [&](Node *Comp) { - if (!Comp) return false; - if (SoFar) SoFar = make<NestedName>(SoFar, Comp); - else SoFar = Comp; - if (State) State->EndsWithTemplateArgs = false; - return SoFar != nullptr; - }; - - if (consumeIf("St")) { - SoFar = make<NameType>("std"); - if (!SoFar) - return nullptr; } + Node *SoFar = nullptr; while (!consumeIf('E')) { - consumeIf('L'); // extension - - // <data-member-prefix> := <member source-name> [<template-args>] M - if (consumeIf('M')) { - if (SoFar == nullptr) - return nullptr; - continue; - } + if (State) + // Only set end-with-template on the case that does that. + State->EndsWithTemplateArgs = false; - // ::= <template-param> if (look() == 'T') { - if (!PushComponent(getDerived().parseTemplateParam())) - return nullptr; - Subs.push_back(SoFar); - continue; - } - - // ::= <template-prefix> <template-args> - if (look() == 'I') { + // ::= <template-param> + if (SoFar != nullptr) + return nullptr; // Cannot have a prefix. + SoFar = getDerived().parseTemplateParam(); + } else if (look() == 'I') { + // ::= <template-prefix> <template-args> + if (SoFar == nullptr) + return nullptr; // Must have a prefix. Node *TA = getDerived().parseTemplateArgs(State != nullptr); - if (TA == nullptr || SoFar == nullptr) - return nullptr; - SoFar = make<NameWithTemplateArgs>(SoFar, TA); - if (!SoFar) - return nullptr; - if (State) State->EndsWithTemplateArgs = true; - Subs.push_back(SoFar); - continue; - } - - // ::= <decltype> - if (look() == 'D' && (look(1) == 't' || look(1) == 'T')) { - if (!PushComponent(getDerived().parseDecltype())) + if (TA == nullptr) return nullptr; - Subs.push_back(SoFar); - continue; - } - - // ::= <substitution> - if (look() == 'S' && look(1) != 't') { - Node *S = getDerived().parseSubstitution(); - if (!PushComponent(S)) + if (SoFar->getKind() == Node::KNameWithTemplateArgs) + // Semantically <template-args> <template-args> cannot be generated by a + // C++ entity. There will always be [something like] a name between + // them. return nullptr; - if (SoFar != S) - Subs.push_back(S); - continue; - } + if (State) + State->EndsWithTemplateArgs = true; + SoFar = make<NameWithTemplateArgs>(SoFar, TA); + } else if (look() == 'D' && (look(1) == 't' || look(1) == 'T')) { + // ::= <decltype> + if (SoFar != nullptr) + return nullptr; // Cannot have a prefix. + SoFar = getDerived().parseDecltype(); + } else { + ModuleName *Module = nullptr; + + if (look() == 'S') { + // ::= <substitution> + Node *S = nullptr; + if (look(1) == 't') { + First += 2; + S = make<NameType>("std"); + } else { + S = getDerived().parseSubstitution(); + } + if (!S) + return nullptr; + if (S->getKind() == Node::KModuleName) { + Module = static_cast<ModuleName *>(S); + } else if (SoFar != nullptr) { + return nullptr; // Cannot have a prefix. + } else { + SoFar = S; + continue; // Do not push a new substitution. + } + } - // Parse an <unqualified-name> thats actually a <ctor-dtor-name>. - if (look() == 'C' || (look() == 'D' && look(1) != 'C')) { - if (SoFar == nullptr) - return nullptr; - if (!PushComponent(getDerived().parseCtorDtorName(SoFar, State))) - return nullptr; - SoFar = getDerived().parseAbiTags(SoFar); - if (SoFar == nullptr) - return nullptr; - Subs.push_back(SoFar); - continue; + // ::= [<prefix>] <unqualified-name> + SoFar = getDerived().parseUnqualifiedName(State, SoFar, Module); } - // ::= <prefix> <unqualified-name> - if (!PushComponent(getDerived().parseUnqualifiedName(State))) + if (SoFar == nullptr) return nullptr; Subs.push_back(SoFar); + + // No longer used. + // <data-member-prefix> := <member source-name> [<template-args>] M + consumeIf('M'); } if (SoFar == nullptr || Subs.empty()) @@ -3365,6 +3329,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseBaseUnresolvedName() { // ::= [gs] <base-unresolved-name> # x or (with "gs") ::x // ::= [gs] sr <unresolved-qualifier-level>+ E <base-unresolved-name> // # A::x, N::y, A<T>::z; "gs" means leading "::" +// [gs] has been parsed by caller. // ::= sr <unresolved-type> <base-unresolved-name> # T::x / decltype(p)::x // extension ::= sr <unresolved-type> <template-args> <base-unresolved-name> // # T::N::x /decltype(p)::N::x @@ -3372,7 +3337,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseBaseUnresolvedName() { // // <unresolved-qualifier-level> ::= <simple-id> template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parseUnresolvedName() { +Node *AbstractManglingParser<Derived, Alloc>::parseUnresolvedName(bool Global) { Node *SoFar = nullptr; // srN <unresolved-type> [<template-args>] <unresolved-qualifier-level>* E <base-unresolved-name> @@ -3406,8 +3371,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseUnresolvedName() { return make<QualifiedName>(SoFar, Base); } - bool Global = consumeIf("gs"); - // [gs] <base-unresolved-name> # x or (with "gs") ::x if (!consumeIf("sr")) { SoFar = getDerived().parseBaseUnresolvedName(); @@ -3637,7 +3600,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseDecltype() { return nullptr; if (!consumeIf('E')) return nullptr; - return make<EnclosingExpr>("decltype(", E, ")"); + return make<EnclosingExpr>("decltype", E); } // <array-type> ::= A <positive dimension number> _ <element type> @@ -3723,8 +3686,8 @@ Node *AbstractManglingParser<Derived, Alloc>::parseQualifiedType() { StringView ProtoSourceName = Qual.dropFront(std::strlen("objcproto")); StringView Proto; { - SwapAndRestore<const char *> SaveFirst(First, ProtoSourceName.begin()), - SaveLast(Last, ProtoSourceName.end()); + ScopedOverride<const char *> SaveFirst(First, ProtoSourceName.begin()), + SaveLast(Last, ProtoSourceName.end()); Proto = parseBareSourceName(); } if (Proto.empty()) @@ -3929,6 +3892,22 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { return nullptr; return make<BinaryFPType>(DimensionNumber); } + // ::= DB <number> _ # C23 signed _BitInt(N) + // ::= DB <instantiation-dependent expression> _ # C23 signed _BitInt(N) + // ::= DU <number> _ # C23 unsigned _BitInt(N) + // ::= DU <instantiation-dependent expression> _ # C23 unsigned _BitInt(N) + case 'B': + case 'U': { + bool Signed = look(1) == 'B'; + First += 2; + Node *Size = std::isdigit(look()) ? make<NameType>(parseNumber()) + : getDerived().parseExpr(); + if (!Size) + return nullptr; + if (!consumeIf('_')) + return nullptr; + return make<BitIntType>(Size, Signed); + } // ::= Di # char32_t case 'i': First += 2; @@ -4077,8 +4056,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { // ::= <substitution> # See Compression below case 'S': { if (look(1) != 't') { - Result = getDerived().parseSubstitution(); - if (Result == nullptr) + bool IsSubst = false; + Result = getDerived().parseUnscopedName(nullptr, &IsSubst); + if (!Result) return nullptr; // Sub could be either of: @@ -4091,12 +4071,14 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { // If this is followed by some <template-args>, and we're permitted to // parse them, take the second production. - if (TryToParseTemplateArgs && look() == 'I') { + if (look() == 'I' && (!IsSubst || TryToParseTemplateArgs)) { + if (!IsSubst) + Subs.push_back(Result); Node *TA = getDerived().parseTemplateArgs(); if (TA == nullptr) return nullptr; Result = make<NameWithTemplateArgs>(Result, TA); - } else { + } else if (IsSubst) { // If all we parsed was a substitution, don't re-insert into the // substitution table. return Result; @@ -4121,22 +4103,24 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { } template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parsePrefixExpr(StringView Kind) { +Node *AbstractManglingParser<Derived, Alloc>::parsePrefixExpr(StringView Kind, + Node::Prec Prec) { Node *E = getDerived().parseExpr(); if (E == nullptr) return nullptr; - return make<PrefixExpr>(Kind, E); + return make<PrefixExpr>(Kind, E, Prec); } template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parseBinaryExpr(StringView Kind) { +Node *AbstractManglingParser<Derived, Alloc>::parseBinaryExpr(StringView Kind, + Node::Prec Prec) { Node *LHS = getDerived().parseExpr(); if (LHS == nullptr) return nullptr; Node *RHS = getDerived().parseExpr(); if (RHS == nullptr) return nullptr; - return make<BinaryExpr>(LHS, Kind, RHS); + return make<BinaryExpr>(LHS, Kind, RHS, Prec); } template <typename Derived, typename Alloc> @@ -4191,43 +4175,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseFunctionParam() { return nullptr; } -// [gs] nw <expression>* _ <type> E # new (expr-list) type -// [gs] nw <expression>* _ <type> <initializer> # new (expr-list) type (init) -// [gs] na <expression>* _ <type> E # new[] (expr-list) type -// [gs] na <expression>* _ <type> <initializer> # new[] (expr-list) type (init) -// <initializer> ::= pi <expression>* E # parenthesized initialization -template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parseNewExpr() { - bool Global = consumeIf("gs"); - bool IsArray = look(1) == 'a'; - if (!consumeIf("nw") && !consumeIf("na")) - return nullptr; - size_t Exprs = Names.size(); - while (!consumeIf('_')) { - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return nullptr; - Names.push_back(Ex); - } - NodeArray ExprList = popTrailingNodeArray(Exprs); - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) - return Ty; - if (consumeIf("pi")) { - size_t InitsBegin = Names.size(); - while (!consumeIf('E')) { - Node *Init = getDerived().parseExpr(); - if (Init == nullptr) - return Init; - Names.push_back(Init); - } - NodeArray Inits = popTrailingNodeArray(InitsBegin); - return make<NewExpr>(ExprList, Ty, Inits, Global, IsArray); - } else if (!consumeIf('E')) - return nullptr; - return make<NewExpr>(ExprList, Ty, NodeArray(), Global, IsArray); -} - // cv <type> <expression> # conversion with one argument // cv <type> _ <expression>* E # conversion with a different number of arguments template <typename Derived, typename Alloc> @@ -4236,7 +4183,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseConversionExpr() { return nullptr; Node *Ty; { - SwapAndRestore<bool> SaveTemp(TryToParseTemplateArgs, false); + ScopedOverride<bool> SaveTemp(TryToParseTemplateArgs, false); Ty = getDerived().parseType(); } @@ -4353,7 +4300,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExprPrimary() { return nullptr; } case 'D': - if (consumeIf("DnE")) + if (consumeIf("Dn") && (consumeIf('0'), consumeIf('E'))) return make<NameType>("nullptr"); return nullptr; case 'T': @@ -4440,55 +4387,38 @@ Node *AbstractManglingParser<Derived, Alloc>::parseFoldExpr() { if (!consumeIf('f')) return nullptr; - char FoldKind = look(); - bool IsLeftFold, HasInitializer; - HasInitializer = FoldKind == 'L' || FoldKind == 'R'; - if (FoldKind == 'l' || FoldKind == 'L') - IsLeftFold = true; - else if (FoldKind == 'r' || FoldKind == 'R') - IsLeftFold = false; - else + bool IsLeftFold = false, HasInitializer = false; + switch (look()) { + default: return nullptr; + case 'L': + IsLeftFold = true; + HasInitializer = true; + break; + case 'R': + HasInitializer = true; + break; + case 'l': + IsLeftFold = true; + break; + case 'r': + break; + } ++First; - // FIXME: This map is duplicated in parseOperatorName and parseExpr. - StringView OperatorName; - if (consumeIf("aa")) OperatorName = "&&"; - else if (consumeIf("an")) OperatorName = "&"; - else if (consumeIf("aN")) OperatorName = "&="; - else if (consumeIf("aS")) OperatorName = "="; - else if (consumeIf("cm")) OperatorName = ","; - else if (consumeIf("ds")) OperatorName = ".*"; - else if (consumeIf("dv")) OperatorName = "/"; - else if (consumeIf("dV")) OperatorName = "/="; - else if (consumeIf("eo")) OperatorName = "^"; - else if (consumeIf("eO")) OperatorName = "^="; - else if (consumeIf("eq")) OperatorName = "=="; - else if (consumeIf("ge")) OperatorName = ">="; - else if (consumeIf("gt")) OperatorName = ">"; - else if (consumeIf("le")) OperatorName = "<="; - else if (consumeIf("ls")) OperatorName = "<<"; - else if (consumeIf("lS")) OperatorName = "<<="; - else if (consumeIf("lt")) OperatorName = "<"; - else if (consumeIf("mi")) OperatorName = "-"; - else if (consumeIf("mI")) OperatorName = "-="; - else if (consumeIf("ml")) OperatorName = "*"; - else if (consumeIf("mL")) OperatorName = "*="; - else if (consumeIf("ne")) OperatorName = "!="; - else if (consumeIf("oo")) OperatorName = "||"; - else if (consumeIf("or")) OperatorName = "|"; - else if (consumeIf("oR")) OperatorName = "|="; - else if (consumeIf("pl")) OperatorName = "+"; - else if (consumeIf("pL")) OperatorName = "+="; - else if (consumeIf("rm")) OperatorName = "%"; - else if (consumeIf("rM")) OperatorName = "%="; - else if (consumeIf("rs")) OperatorName = ">>"; - else if (consumeIf("rS")) OperatorName = ">>="; - else return nullptr; - - Node *Pack = getDerived().parseExpr(), *Init = nullptr; + const auto *Op = parseOperatorEncoding(); + if (!Op) + return nullptr; + if (!(Op->getKind() == OperatorInfo::Binary + || (Op->getKind() == OperatorInfo::Member + && Op->getName().back() == '*'))) + return nullptr; + + Node *Pack = getDerived().parseExpr(); if (Pack == nullptr) return nullptr; + + Node *Init = nullptr; if (HasInitializer) { Init = getDerived().parseExpr(); if (Init == nullptr) @@ -4498,14 +4428,16 @@ Node *AbstractManglingParser<Derived, Alloc>::parseFoldExpr() { if (IsLeftFold && Init) std::swap(Pack, Init); - return make<FoldExpr>(IsLeftFold, OperatorName, Pack, Init); + return make<FoldExpr>(IsLeftFold, Op->getSymbol(), Pack, Init); } // <expression> ::= mc <parameter type> <expr> [<offset number>] E // // Not yet in the spec: https://github.com/itanium-cxx-abi/cxx-abi/issues/47 template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parsePointerToMemberConversionExpr() { +Node * +AbstractManglingParser<Derived, Alloc>::parsePointerToMemberConversionExpr( + Node::Prec Prec) { Node *Ty = getDerived().parseType(); if (!Ty) return nullptr; @@ -4515,7 +4447,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parsePointerToMemberConversionExpr StringView Offset = getDerived().parseNumber(true); if (!consumeIf('E')) return nullptr; - return make<PointerToMemberConversionExpr>(Ty, Expr, Offset); + return make<PointerToMemberConversionExpr>(Ty, Expr, Offset, Prec); } // <expression> ::= so <referent type> <expr> [<offset number>] <union-selector>* [p] E @@ -4592,316 +4524,127 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSubobjectExpr() { template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { bool Global = consumeIf("gs"); - if (numLeft() < 2) - return nullptr; - switch (*First) { - case 'L': - return getDerived().parseExprPrimary(); - case 'T': - return getDerived().parseTemplateParam(); - case 'f': { - // Disambiguate a fold expression from a <function-param>. - if (look(1) == 'p' || (look(1) == 'L' && std::isdigit(look(2)))) - return getDerived().parseFunctionParam(); - return getDerived().parseFoldExpr(); - } - case 'a': - switch (First[1]) { - case 'a': - First += 2; - return getDerived().parseBinaryExpr("&&"); - case 'd': - First += 2; - return getDerived().parsePrefixExpr("&"); - case 'n': - First += 2; - return getDerived().parseBinaryExpr("&"); - case 'N': - First += 2; - return getDerived().parseBinaryExpr("&="); - case 'S': - First += 2; - return getDerived().parseBinaryExpr("="); - case 't': { - First += 2; - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) - return nullptr; - return make<EnclosingExpr>("alignof (", Ty, ")"); - } - case 'z': { - First += 2; - Node *Ty = getDerived().parseExpr(); - if (Ty == nullptr) - return nullptr; - return make<EnclosingExpr>("alignof (", Ty, ")"); - } - } - return nullptr; - case 'c': - switch (First[1]) { - // cc <type> <expression> # const_cast<type>(expression) - case 'c': { - First += 2; - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) - return Ty; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<CastExpr>("const_cast", Ty, Ex); - } - // cl <expression>+ E # call - case 'l': { - First += 2; - Node *Callee = getDerived().parseExpr(); - if (Callee == nullptr) - return Callee; - size_t ExprsBegin = Names.size(); - while (!consumeIf('E')) { - Node *E = getDerived().parseExpr(); - if (E == nullptr) - return E; - Names.push_back(E); - } - return make<CallExpr>(Callee, popTrailingNodeArray(ExprsBegin)); - } - case 'm': - First += 2; - return getDerived().parseBinaryExpr(","); - case 'o': - First += 2; - return getDerived().parsePrefixExpr("~"); - case 'v': - return getDerived().parseConversionExpr(); - } - return nullptr; - case 'd': - switch (First[1]) { - case 'a': { - First += 2; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<DeleteExpr>(Ex, Global, /*is_array=*/true); - } - case 'c': { - First += 2; - Node *T = getDerived().parseType(); - if (T == nullptr) - return T; + const auto *Op = parseOperatorEncoding(); + if (Op) { + auto Sym = Op->getSymbol(); + switch (Op->getKind()) { + case OperatorInfo::Binary: + // Binary operator: lhs @ rhs + return getDerived().parseBinaryExpr(Sym, Op->getPrecedence()); + case OperatorInfo::Prefix: + // Prefix unary operator: @ expr + return getDerived().parsePrefixExpr(Sym, Op->getPrecedence()); + case OperatorInfo::Postfix: { + // Postfix unary operator: expr @ + if (consumeIf('_')) + return getDerived().parsePrefixExpr(Sym, Op->getPrecedence()); Node *Ex = getDerived().parseExpr(); if (Ex == nullptr) - return Ex; - return make<CastExpr>("dynamic_cast", T, Ex); - } - case 'e': - First += 2; - return getDerived().parsePrefixExpr("*"); - case 'l': { - First += 2; - Node *E = getDerived().parseExpr(); - if (E == nullptr) - return E; - return make<DeleteExpr>(E, Global, /*is_array=*/false); + return nullptr; + return make<PostfixExpr>(Ex, Sym, Op->getPrecedence()); } - case 'n': - return getDerived().parseUnresolvedName(); - case 's': { - First += 2; - Node *LHS = getDerived().parseExpr(); - if (LHS == nullptr) + case OperatorInfo::Array: { + // Array Index: lhs [ rhs ] + Node *Base = getDerived().parseExpr(); + if (Base == nullptr) return nullptr; - Node *RHS = getDerived().parseExpr(); - if (RHS == nullptr) + Node *Index = getDerived().parseExpr(); + if (Index == nullptr) return nullptr; - return make<MemberExpr>(LHS, ".*", RHS); + return make<ArraySubscriptExpr>(Base, Index, Op->getPrecedence()); } - case 't': { - First += 2; + case OperatorInfo::Member: { + // Member access lhs @ rhs Node *LHS = getDerived().parseExpr(); if (LHS == nullptr) - return LHS; + return nullptr; Node *RHS = getDerived().parseExpr(); if (RHS == nullptr) return nullptr; - return make<MemberExpr>(LHS, ".", RHS); - } - case 'v': - First += 2; - return getDerived().parseBinaryExpr("/"); - case 'V': - First += 2; - return getDerived().parseBinaryExpr("/="); - } - return nullptr; - case 'e': - switch (First[1]) { - case 'o': - First += 2; - return getDerived().parseBinaryExpr("^"); - case 'O': - First += 2; - return getDerived().parseBinaryExpr("^="); - case 'q': - First += 2; - return getDerived().parseBinaryExpr("=="); - } - return nullptr; - case 'g': - switch (First[1]) { - case 'e': - First += 2; - return getDerived().parseBinaryExpr(">="); - case 't': - First += 2; - return getDerived().parseBinaryExpr(">"); - } - return nullptr; - case 'i': - switch (First[1]) { - case 'x': { - First += 2; - Node *Base = getDerived().parseExpr(); - if (Base == nullptr) + return make<MemberExpr>(LHS, Sym, RHS, Op->getPrecedence()); + } + case OperatorInfo::New: { + // New + // # new (expr-list) type [(init)] + // [gs] nw <expression>* _ <type> [pi <expression>*] E + // # new[] (expr-list) type [(init)] + // [gs] na <expression>* _ <type> [pi <expression>*] E + size_t Exprs = Names.size(); + while (!consumeIf('_')) { + Node *Ex = getDerived().parseExpr(); + if (Ex == nullptr) + return nullptr; + Names.push_back(Ex); + } + NodeArray ExprList = popTrailingNodeArray(Exprs); + Node *Ty = getDerived().parseType(); + if (Ty == nullptr) return nullptr; - Node *Index = getDerived().parseExpr(); - if (Index == nullptr) - return Index; - return make<ArraySubscriptExpr>(Base, Index); - } - case 'l': { - First += 2; + bool HaveInits = consumeIf("pi"); size_t InitsBegin = Names.size(); while (!consumeIf('E')) { - Node *E = getDerived().parseBracedExpr(); - if (E == nullptr) + if (!HaveInits) return nullptr; - Names.push_back(E); + Node *Init = getDerived().parseExpr(); + if (Init == nullptr) + return Init; + Names.push_back(Init); } - return make<InitListExpr>(nullptr, popTrailingNodeArray(InitsBegin)); + NodeArray Inits = popTrailingNodeArray(InitsBegin); + return make<NewExpr>(ExprList, Ty, Inits, Global, + /*IsArray=*/Op->getFlag(), Op->getPrecedence()); } - } - return nullptr; - case 'l': - switch (First[1]) { - case 'e': - First += 2; - return getDerived().parseBinaryExpr("<="); - case 's': - First += 2; - return getDerived().parseBinaryExpr("<<"); - case 'S': - First += 2; - return getDerived().parseBinaryExpr("<<="); - case 't': - First += 2; - return getDerived().parseBinaryExpr("<"); - } - return nullptr; - case 'm': - switch (First[1]) { - case 'c': - First += 2; - return parsePointerToMemberConversionExpr(); - case 'i': - First += 2; - return getDerived().parseBinaryExpr("-"); - case 'I': - First += 2; - return getDerived().parseBinaryExpr("-="); - case 'l': - First += 2; - return getDerived().parseBinaryExpr("*"); - case 'L': - First += 2; - return getDerived().parseBinaryExpr("*="); - case 'm': - First += 2; - if (consumeIf('_')) - return getDerived().parsePrefixExpr("--"); + case OperatorInfo::Del: { + // Delete Node *Ex = getDerived().parseExpr(); if (Ex == nullptr) return nullptr; - return make<PostfixExpr>(Ex, "--"); - } - return nullptr; - case 'n': - switch (First[1]) { - case 'a': - case 'w': - return getDerived().parseNewExpr(); - case 'e': - First += 2; - return getDerived().parseBinaryExpr("!="); - case 'g': - First += 2; - return getDerived().parsePrefixExpr("-"); - case 't': - First += 2; - return getDerived().parsePrefixExpr("!"); - case 'x': - First += 2; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<EnclosingExpr>("noexcept (", Ex, ")"); + return make<DeleteExpr>(Ex, Global, /*IsArray=*/Op->getFlag(), + Op->getPrecedence()); } - return nullptr; - case 'o': - switch (First[1]) { - case 'n': - return getDerived().parseUnresolvedName(); - case 'o': - First += 2; - return getDerived().parseBinaryExpr("||"); - case 'r': - First += 2; - return getDerived().parseBinaryExpr("|"); - case 'R': - First += 2; - return getDerived().parseBinaryExpr("|="); - } - return nullptr; - case 'p': - switch (First[1]) { - case 'm': - First += 2; - return getDerived().parseBinaryExpr("->*"); - case 'l': - First += 2; - return getDerived().parseBinaryExpr("+"); - case 'L': - First += 2; - return getDerived().parseBinaryExpr("+="); - case 'p': { - First += 2; - if (consumeIf('_')) - return getDerived().parsePrefixExpr("++"); - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<PostfixExpr>(Ex, "++"); + case OperatorInfo::Call: { + // Function Call + Node *Callee = getDerived().parseExpr(); + if (Callee == nullptr) + return nullptr; + size_t ExprsBegin = Names.size(); + while (!consumeIf('E')) { + Node *E = getDerived().parseExpr(); + if (E == nullptr) + return nullptr; + Names.push_back(E); + } + return make<CallExpr>(Callee, popTrailingNodeArray(ExprsBegin), + Op->getPrecedence()); } - case 's': - First += 2; - return getDerived().parsePrefixExpr("+"); - case 't': { - First += 2; - Node *L = getDerived().parseExpr(); - if (L == nullptr) + case OperatorInfo::CCast: { + // C Cast: (type)expr + Node *Ty; + { + ScopedOverride<bool> SaveTemp(TryToParseTemplateArgs, false); + Ty = getDerived().parseType(); + } + if (Ty == nullptr) return nullptr; - Node *R = getDerived().parseExpr(); - if (R == nullptr) + + size_t ExprsBegin = Names.size(); + bool IsMany = consumeIf('_'); + while (!consumeIf('E')) { + Node *E = getDerived().parseExpr(); + if (E == nullptr) + return E; + Names.push_back(E); + if (!IsMany) + break; + } + NodeArray Exprs = popTrailingNodeArray(ExprsBegin); + if (!IsMany && Exprs.size() != 1) return nullptr; - return make<MemberExpr>(L, "->", R); - } + return make<ConversionExpr>(Ty, Exprs, Op->getPrecedence()); } - return nullptr; - case 'q': - if (First[1] == 'u') { - First += 2; + case OperatorInfo::Conditional: { + // Conditional operator: expr ? expr : expr Node *Cond = getDerived().parseExpr(); if (Cond == nullptr) return nullptr; @@ -4911,147 +4654,120 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { Node *RHS = getDerived().parseExpr(); if (RHS == nullptr) return nullptr; - return make<ConditionalExpr>(Cond, LHS, RHS); - } - return nullptr; - case 'r': - switch (First[1]) { - case 'c': { - First += 2; - Node *T = getDerived().parseType(); - if (T == nullptr) - return T; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<CastExpr>("reinterpret_cast", T, Ex); + return make<ConditionalExpr>(Cond, LHS, RHS, Op->getPrecedence()); } - case 'm': - First += 2; - return getDerived().parseBinaryExpr("%"); - case 'M': - First += 2; - return getDerived().parseBinaryExpr("%="); - case 's': - First += 2; - return getDerived().parseBinaryExpr(">>"); - case 'S': - First += 2; - return getDerived().parseBinaryExpr(">>="); - } - return nullptr; - case 's': - switch (First[1]) { - case 'c': { - First += 2; - Node *T = getDerived().parseType(); - if (T == nullptr) - return T; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<CastExpr>("static_cast", T, Ex); - } - case 'o': - First += 2; - return parseSubobjectExpr(); - case 'p': { - First += 2; - Node *Child = getDerived().parseExpr(); - if (Child == nullptr) - return nullptr; - return make<ParameterPackExpansion>(Child); - } - case 'r': - return getDerived().parseUnresolvedName(); - case 't': { - First += 2; + case OperatorInfo::NamedCast: { + // Named cast operation, @<type>(expr) Node *Ty = getDerived().parseType(); if (Ty == nullptr) - return Ty; - return make<EnclosingExpr>("sizeof (", Ty, ")"); - } - case 'z': { - First += 2; + return nullptr; Node *Ex = getDerived().parseExpr(); if (Ex == nullptr) - return Ex; - return make<EnclosingExpr>("sizeof (", Ex, ")"); + return nullptr; + return make<CastExpr>(Sym, Ty, Ex, Op->getPrecedence()); } - case 'Z': - First += 2; - if (look() == 'T') { - Node *R = getDerived().parseTemplateParam(); - if (R == nullptr) - return nullptr; - return make<SizeofParamPackExpr>(R); - } else if (look() == 'f') { - Node *FP = getDerived().parseFunctionParam(); - if (FP == nullptr) - return nullptr; - return make<EnclosingExpr>("sizeof... (", FP, ")"); - } - return nullptr; - case 'P': { - First += 2; - size_t ArgsBegin = Names.size(); - while (!consumeIf('E')) { - Node *Arg = getDerived().parseTemplateArg(); - if (Arg == nullptr) - return nullptr; - Names.push_back(Arg); - } - auto *Pack = make<NodeArrayNode>(popTrailingNodeArray(ArgsBegin)); - if (!Pack) + case OperatorInfo::OfIdOp: { + // [sizeof/alignof/typeid] ( <type>|<expr> ) + Node *Arg = + Op->getFlag() ? getDerived().parseType() : getDerived().parseExpr(); + if (!Arg) return nullptr; - return make<EnclosingExpr>("sizeof... (", Pack, ")"); + return make<EnclosingExpr>(Sym, Arg, Op->getPrecedence()); } + case OperatorInfo::NameOnly: { + // Not valid as an expression operand. + return nullptr; } - return nullptr; - case 't': - switch (First[1]) { - case 'e': { - First += 2; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) - return Ex; - return make<EnclosingExpr>("typeid (", Ex, ")"); } - case 'i': { - First += 2; - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) - return Ty; - return make<EnclosingExpr>("typeid (", Ty, ")"); + DEMANGLE_UNREACHABLE; + } + + if (numLeft() < 2) + return nullptr; + + if (look() == 'L') + return getDerived().parseExprPrimary(); + if (look() == 'T') + return getDerived().parseTemplateParam(); + if (look() == 'f') { + // Disambiguate a fold expression from a <function-param>. + if (look(1) == 'p' || (look(1) == 'L' && std::isdigit(look(2)))) + return getDerived().parseFunctionParam(); + return getDerived().parseFoldExpr(); + } + if (consumeIf("il")) { + size_t InitsBegin = Names.size(); + while (!consumeIf('E')) { + Node *E = getDerived().parseBracedExpr(); + if (E == nullptr) + return nullptr; + Names.push_back(E); } - case 'l': { - First += 2; - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) + return make<InitListExpr>(nullptr, popTrailingNodeArray(InitsBegin)); + } + if (consumeIf("mc")) + return parsePointerToMemberConversionExpr(Node::Prec::Unary); + if (consumeIf("nx")) { + Node *Ex = getDerived().parseExpr(); + if (Ex == nullptr) + return Ex; + return make<EnclosingExpr>("noexcept ", Ex, Node::Prec::Unary); + } + if (consumeIf("so")) + return parseSubobjectExpr(); + if (consumeIf("sp")) { + Node *Child = getDerived().parseExpr(); + if (Child == nullptr) + return nullptr; + return make<ParameterPackExpansion>(Child); + } + if (consumeIf("sZ")) { + if (look() == 'T') { + Node *R = getDerived().parseTemplateParam(); + if (R == nullptr) return nullptr; - size_t InitsBegin = Names.size(); - while (!consumeIf('E')) { - Node *E = getDerived().parseBracedExpr(); - if (E == nullptr) - return nullptr; - Names.push_back(E); - } - return make<InitListExpr>(Ty, popTrailingNodeArray(InitsBegin)); + return make<SizeofParamPackExpr>(R); } - case 'r': - First += 2; - return make<NameType>("throw"); - case 'w': { - First += 2; - Node *Ex = getDerived().parseExpr(); - if (Ex == nullptr) + Node *FP = getDerived().parseFunctionParam(); + if (FP == nullptr) + return nullptr; + return make<EnclosingExpr>("sizeof... ", FP); + } + if (consumeIf("sP")) { + size_t ArgsBegin = Names.size(); + while (!consumeIf('E')) { + Node *Arg = getDerived().parseTemplateArg(); + if (Arg == nullptr) return nullptr; - return make<ThrowExpr>(Ex); + Names.push_back(Arg); } + auto *Pack = make<NodeArrayNode>(popTrailingNodeArray(ArgsBegin)); + if (!Pack) + return nullptr; + return make<EnclosingExpr>("sizeof... ", Pack); + } + if (consumeIf("tl")) { + Node *Ty = getDerived().parseType(); + if (Ty == nullptr) + return nullptr; + size_t InitsBegin = Names.size(); + while (!consumeIf('E')) { + Node *E = getDerived().parseBracedExpr(); + if (E == nullptr) + return nullptr; + Names.push_back(E); } - return nullptr; - case 'u': { - ++First; + return make<InitListExpr>(Ty, popTrailingNodeArray(InitsBegin)); + } + if (consumeIf("tr")) + return make<NameType>("throw"); + if (consumeIf("tw")) { + Node *Ex = getDerived().parseExpr(); + if (Ex == nullptr) + return nullptr; + return make<ThrowExpr>(Ex); + } + if (consumeIf('u')) { Node *Name = getDerived().parseSourceName(/*NameState=*/nullptr); if (!Name) return nullptr; @@ -5060,45 +4776,36 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { // interpreted as <type> node 'short' or 'ellipsis'. However, neither // __uuidof(short) nor __uuidof(...) can actually appear, so there is no // actual conflict here. + bool IsUUID = false; + Node *UUID = nullptr; if (Name->getBaseName() == "__uuidof") { - if (numLeft() < 2) - return nullptr; - if (*First == 't') { - ++First; - Node *Ty = getDerived().parseType(); - if (!Ty) - return nullptr; - return make<CallExpr>(Name, makeNodeArray(&Ty, &Ty + 1)); - } - if (*First == 'z') { - ++First; - Node *Ex = getDerived().parseExpr(); - if (!Ex) - return nullptr; - return make<CallExpr>(Name, makeNodeArray(&Ex, &Ex + 1)); + if (consumeIf('t')) { + UUID = getDerived().parseType(); + IsUUID = true; + } else if (consumeIf('z')) { + UUID = getDerived().parseExpr(); + IsUUID = true; } } size_t ExprsBegin = Names.size(); - while (!consumeIf('E')) { - Node *E = getDerived().parseTemplateArg(); - if (E == nullptr) - return E; - Names.push_back(E); + if (IsUUID) { + if (UUID == nullptr) + return nullptr; + Names.push_back(UUID); + } else { + while (!consumeIf('E')) { + Node *E = getDerived().parseTemplateArg(); + if (E == nullptr) + return E; + Names.push_back(E); + } } - return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin)); - } - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return getDerived().parseUnresolvedName(); + return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin), + Node::Prec::Postfix); } - return nullptr; + + // Only unresolved names remain. + return getDerived().parseUnresolvedName(Global); } // <call-offset> ::= h <nv-offset> _ @@ -5131,14 +4838,17 @@ bool AbstractManglingParser<Alloc, Derived>::parseCallOffset() { // # second call-offset is result adjustment // ::= T <call-offset> <base encoding> // # base is the nominal target function of thunk -// ::= GV <object name> # Guard variable for one-time initialization +// # Guard variable for one-time initialization +// ::= GV <object name> // # No <type> // ::= TW <object name> # Thread-local wrapper // ::= TH <object name> # Thread-local initialization // ::= GR <object name> _ # First temporary // ::= GR <object name> <seq-id> _ # Subsequent temporaries -// extension ::= TC <first type> <number> _ <second type> # construction vtable for second-in-first +// # construction vtable for second-in-first +// extension ::= TC <first type> <number> _ <second type> // extension ::= GR <object name> # reference temporary for object +// extension ::= GI <module name> # module global initializer template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() { switch (look()) { @@ -5265,6 +4975,16 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() { return nullptr; return make<SpecialName>("reference temporary for ", Name); } + // GI <module-name> v + case 'I': { + First += 2; + ModuleName *Module = nullptr; + if (getDerived().parseModuleNameOpt(Module)) + return nullptr; + if (Module == nullptr) + return nullptr; + return make<SpecialName>("initializer for module ", Module); + } } } return nullptr; @@ -5379,7 +5099,7 @@ template <> struct FloatData<long double> { #if defined(__mips__) && defined(__mips_n64) || defined(__aarch64__) || \ - defined(__wasm__) + defined(__wasm__) || defined(__riscv) static const size_t mangled_size = 32; #elif defined(__arm__) || defined(__mips__) || defined(__hexagon__) static const size_t mangled_size = 16; @@ -5444,6 +5164,7 @@ bool AbstractManglingParser<Alloc, Derived>::parseSeqId(size_t *Out) { // <substitution> ::= Si # ::std::basic_istream<char, std::char_traits<char> > // <substitution> ::= So # ::std::basic_ostream<char, std::char_traits<char> > // <substitution> ::= Sd # ::std::basic_iostream<char, std::char_traits<char> > +// The St case is handled specially in parseNestedName. template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseSubstitution() { if (!consumeIf('S')) diff --git a/llvm/include/llvm/Demangle/ItaniumNodes.def b/llvm/include/llvm/Demangle/ItaniumNodes.def new file mode 100644 index 000000000000..c0e277d554cc --- /dev/null +++ b/llvm/include/llvm/Demangle/ItaniumNodes.def @@ -0,0 +1,95 @@ +//===--- ItaniumNodes.def ------------*- mode:c++;eval:(read-only-mode) -*-===// +// Do not edit! See README.txt. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Define the demangler's node names + +#ifndef NODE +#error Define NODE to handle nodes +#endif + +NODE(NodeArrayNode) +NODE(DotSuffix) +NODE(VendorExtQualType) +NODE(QualType) +NODE(ConversionOperatorType) +NODE(PostfixQualifiedType) +NODE(ElaboratedTypeSpefType) +NODE(NameType) +NODE(AbiTagAttr) +NODE(EnableIfAttr) +NODE(ObjCProtoName) +NODE(PointerType) +NODE(ReferenceType) +NODE(PointerToMemberType) +NODE(ArrayType) +NODE(FunctionType) +NODE(NoexceptSpec) +NODE(DynamicExceptionSpec) +NODE(FunctionEncoding) +NODE(LiteralOperator) +NODE(SpecialName) +NODE(CtorVtableSpecialName) +NODE(QualifiedName) +NODE(NestedName) +NODE(LocalName) +NODE(ModuleName) +NODE(ModuleEntity) +NODE(VectorType) +NODE(PixelVectorType) +NODE(BinaryFPType) +NODE(BitIntType) +NODE(SyntheticTemplateParamName) +NODE(TypeTemplateParamDecl) +NODE(NonTypeTemplateParamDecl) +NODE(TemplateTemplateParamDecl) +NODE(TemplateParamPackDecl) +NODE(ParameterPack) +NODE(TemplateArgumentPack) +NODE(ParameterPackExpansion) +NODE(TemplateArgs) +NODE(ForwardTemplateReference) +NODE(NameWithTemplateArgs) +NODE(GlobalQualifiedName) +NODE(ExpandedSpecialSubstitution) +NODE(SpecialSubstitution) +NODE(CtorDtorName) +NODE(DtorName) +NODE(UnnamedTypeName) +NODE(ClosureTypeName) +NODE(StructuredBindingName) +NODE(BinaryExpr) +NODE(ArraySubscriptExpr) +NODE(PostfixExpr) +NODE(ConditionalExpr) +NODE(MemberExpr) +NODE(SubobjectExpr) +NODE(EnclosingExpr) +NODE(CastExpr) +NODE(SizeofParamPackExpr) +NODE(CallExpr) +NODE(NewExpr) +NODE(DeleteExpr) +NODE(PrefixExpr) +NODE(FunctionParam) +NODE(ConversionExpr) +NODE(PointerToMemberConversionExpr) +NODE(InitListExpr) +NODE(FoldExpr) +NODE(ThrowExpr) +NODE(BoolExpr) +NODE(StringLiteral) +NODE(LambdaExpr) +NODE(EnumLiteral) +NODE(IntegerLiteral) +NODE(FloatLiteral) +NODE(DoubleLiteral) +NODE(LongDoubleLiteral) +NODE(BracedExpr) +NODE(BracedRangeExpr) + +#undef NODE diff --git a/llvm/include/llvm/Demangle/Utility.h b/llvm/include/llvm/Demangle/Utility.h index 1cf7e8f1df45..ca7e44b948c7 100644 --- a/llvm/include/llvm/Demangle/Utility.h +++ b/llvm/include/llvm/Demangle/Utility.h @@ -33,43 +33,50 @@ class OutputBuffer { size_t CurrentPosition = 0; size_t BufferCapacity = 0; - // Ensure there is at least n more positions in buffer. + // Ensure there are at least N more positions in the buffer. void grow(size_t N) { - if (N + CurrentPosition >= BufferCapacity) { + size_t Need = N + CurrentPosition; + if (Need > BufferCapacity) { + // Reduce the number of reallocations, with a bit of hysteresis. The + // number here is chosen so the first allocation will more-than-likely not + // allocate more than 1K. + Need += 1024 - 32; BufferCapacity *= 2; - if (BufferCapacity < N + CurrentPosition) - BufferCapacity = N + CurrentPosition; + if (BufferCapacity < Need) + BufferCapacity = Need; Buffer = static_cast<char *>(std::realloc(Buffer, BufferCapacity)); if (Buffer == nullptr) std::terminate(); } } - void writeUnsigned(uint64_t N, bool isNeg = false) { - // Handle special case... - if (N == 0) { - *this << '0'; - return; - } - + OutputBuffer &writeUnsigned(uint64_t N, bool isNeg = false) { std::array<char, 21> Temp; char *TempPtr = Temp.data() + Temp.size(); - while (N) { + // Output at least one character. + do { *--TempPtr = char('0' + N % 10); N /= 10; - } + } while (N); - // Add negative sign... + // Add negative sign. if (isNeg) *--TempPtr = '-'; - this->operator<<(StringView(TempPtr, Temp.data() + Temp.size())); + + return operator+=(StringView(TempPtr, Temp.data() + Temp.size())); } public: OutputBuffer(char *StartBuf, size_t Size) : Buffer(StartBuf), CurrentPosition(0), BufferCapacity(Size) {} OutputBuffer() = default; + // Non-copyable + OutputBuffer(const OutputBuffer &) = delete; + OutputBuffer &operator=(const OutputBuffer &) = delete; + + operator StringView() const { return StringView(Buffer, CurrentPosition); } + void reset(char *Buffer_, size_t BufferCapacity_) { CurrentPosition = 0; Buffer = Buffer_; @@ -81,13 +88,27 @@ public: unsigned CurrentPackIndex = std::numeric_limits<unsigned>::max(); unsigned CurrentPackMax = std::numeric_limits<unsigned>::max(); + /// When zero, we're printing template args and '>' needs to be parenthesized. + /// Use a counter so we can simply increment inside parentheses. + unsigned GtIsGt = 1; + + bool isGtInsideTemplateArgs() const { return GtIsGt == 0; } + + void printOpen(char Open = '(') { + GtIsGt++; + *this += Open; + } + void printClose(char Close = ')') { + GtIsGt--; + *this += Close; + } + OutputBuffer &operator+=(StringView R) { - size_t Size = R.size(); - if (Size == 0) - return *this; - grow(Size); - std::memmove(Buffer + CurrentPosition, R.begin(), Size); - CurrentPosition += Size; + if (size_t Size = R.size()) { + grow(Size); + std::memcpy(Buffer + CurrentPosition, R.begin(), Size); + CurrentPosition += Size; + } return *this; } @@ -97,9 +118,7 @@ public: return *this; } - OutputBuffer &operator<<(StringView R) { return (*this += R); } - - OutputBuffer prepend(StringView R) { + OutputBuffer &prepend(StringView R) { size_t Size = R.size(); grow(Size); @@ -110,19 +129,16 @@ public: return *this; } + OutputBuffer &operator<<(StringView R) { return (*this += R); } + OutputBuffer &operator<<(char C) { return (*this += C); } OutputBuffer &operator<<(long long N) { - if (N < 0) - writeUnsigned(static_cast<unsigned long long>(-N), true); - else - writeUnsigned(static_cast<unsigned long long>(N)); - return *this; + return writeUnsigned(static_cast<unsigned long long>(std::abs(N)), N < 0); } OutputBuffer &operator<<(unsigned long long N) { - writeUnsigned(N, false); - return *this; + return writeUnsigned(N, false); } OutputBuffer &operator<<(long N) { @@ -155,7 +171,8 @@ public: void setCurrentPosition(size_t NewPos) { CurrentPosition = NewPos; } char back() const { - return CurrentPosition ? Buffer[CurrentPosition - 1] : '\0'; + assert(CurrentPosition); + return Buffer[CurrentPosition - 1]; } bool empty() const { return CurrentPosition == 0; } @@ -165,35 +182,20 @@ public: size_t getBufferCapacity() const { return BufferCapacity; } }; -template <class T> class SwapAndRestore { - T &Restore; - T OriginalValue; - bool ShouldRestore = true; +template <class T> class ScopedOverride { + T &Loc; + T Original; public: - SwapAndRestore(T &Restore_) : SwapAndRestore(Restore_, Restore_) {} - - SwapAndRestore(T &Restore_, T NewVal) - : Restore(Restore_), OriginalValue(Restore) { - Restore = std::move(NewVal); - } - ~SwapAndRestore() { - if (ShouldRestore) - Restore = std::move(OriginalValue); - } - - void shouldRestore(bool ShouldRestore_) { ShouldRestore = ShouldRestore_; } - - void restoreNow(bool Force) { - if (!Force && !ShouldRestore) - return; + ScopedOverride(T &Loc_) : ScopedOverride(Loc_, Loc_) {} - Restore = std::move(OriginalValue); - ShouldRestore = false; + ScopedOverride(T &Loc_, T NewVal) : Loc(Loc_), Original(Loc_) { + Loc_ = std::move(NewVal); } + ~ScopedOverride() { Loc = std::move(Original); } - SwapAndRestore(const SwapAndRestore &) = delete; - SwapAndRestore &operator=(const SwapAndRestore &) = delete; + ScopedOverride(const ScopedOverride &) = delete; + ScopedOverride &operator=(const ScopedOverride &) = delete; }; inline bool initializeOutputBuffer(char *Buf, size_t *N, OutputBuffer &OB, diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h new file mode 100644 index 000000000000..d748d4b0fa59 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h @@ -0,0 +1,35 @@ +//===--------- DWARFRecordSectionSplitter.h - JITLink -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H +#define LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// A LinkGraph pass that splits blocks in a section that follows the DWARF +/// Record format into sub-blocks where each header gets its own block. +/// When splitting EHFrames, DWARFRecordSectionSplitter should not be run +/// without EHFrameEdgeFixer, which is responsible for adding FDE-to-CIE edges. +class DWARFRecordSectionSplitter { +public: + DWARFRecordSectionSplitter(StringRef SectionName); + Error operator()(LinkGraph &G); + +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef SectionName; +}; + +} // namespace jitlink +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 25f1349f15f2..897808c0ee83 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -223,6 +223,11 @@ public: /// Returns the size of this defined addressable. size_t getSize() const { return Size; } + /// Returns the address range of this defined addressable. + orc::ExecutorAddrRange getRange() const { + return orc::ExecutorAddrRange(getAddress(), getSize()); + } + /// Get the content for this block. Block must not be a zero-fill block. ArrayRef<char> getContent() const { assert(Data && "Block does not contain content"); @@ -576,6 +581,11 @@ public: this->Size = Size; } + /// Returns the address range of this symbol. + orc::ExecutorAddrRange getRange() const { + return orc::ExecutorAddrRange(getAddress(), getSize()); + } + /// Returns true if this symbol is backed by a zero-fill block. /// This method may only be called on defined symbols. bool isSymbolZeroFill() const { return getBlock().isZeroFill(); } @@ -1215,8 +1225,11 @@ public: /// Make the given symbol an absolute with the given address (must not already /// be absolute). /// - /// Symbol size, linkage, scope, and callability, and liveness will be left - /// unchanged. Symbol offset will be reset to 0. + /// The symbol's size, linkage, and callability, and liveness will be left + /// unchanged, and its offset will be reset to 0. + /// + /// If the symbol was external then its scope will be set to local, otherwise + /// it will be left unchanged. void makeAbsolute(Symbol &Sym, orc::ExecutorAddr Address) { assert(!Sym.isAbsolute() && "Symbol is already absolute"); if (Sym.isExternal()) { @@ -1225,6 +1238,7 @@ public: assert(Sym.getOffset() == 0 && "External is not at offset 0"); ExternalSymbols.erase(&Sym); Sym.getAddressable().setAbsolute(true); + Sym.setScope(Scope::Local); } else { assert(Sym.isDefined() && "Sym is not a defined symbol"); Section &Sec = Sym.getBlock().getSection(); @@ -1733,6 +1747,9 @@ Error markAllSymbolsLive(LinkGraph &G); Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, const Edge &E); +Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N, + const Edge &E); + /// Base case for edge-visitors where the visitor-list is empty. inline void visitEdge(LinkGraph &G, Block *B, Edge &E) {} diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h index aee14c0d1fe5..6f2ff012697d 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -18,30 +18,6 @@ namespace llvm { namespace jitlink { -namespace MachO_arm64_Edges { - -enum MachOARM64RelocationKind : Edge::Kind { - Branch26 = Edge::FirstRelocation, - Pointer32, - Pointer64, - Pointer64Anon, - Page21, - PageOffset12, - GOTPage21, - GOTPageOffset12, - TLVPage21, - TLVPageOffset12, - PointerToGOT, - PairedAddend, - LDRLiteral19, - Delta32, - Delta64, - NegDelta32, - NegDelta64, -}; - -} // namespace MachO_arm64_Edges - /// Create a LinkGraph from a MachO/arm64 relocatable object. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy @@ -62,9 +38,6 @@ createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer); void link_MachO_arm64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx); -/// Return the string name of the given MachO arm64 edge kind. -const char *getMachOARM64RelocationKindName(Edge::Kind R); - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h b/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h index e9771319ef06..a18098e5a1a9 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h @@ -152,13 +152,9 @@ public: using iterator = typename VectorTy::iterator; AllocGroupSmallMap() = default; - AllocGroupSmallMap(std::initializer_list<std::pair<AllocGroup, T>> Inits) { - Elems.reserve(Inits.size()); - for (const auto &E : Inits) - Elems.push_back(E); - llvm::sort(Elems, [](const ElemT &LHS, const ElemT &RHS) { - return LHS.first < RHS.first; - }); + AllocGroupSmallMap(std::initializer_list<std::pair<AllocGroup, T>> Inits) + : Elems(Inits) { + llvm::sort(Elems, llvm::less_first()); } iterator begin() { return Elems.begin(); } diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 994ce783b058..53ff6c7a219e 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -13,24 +13,353 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H #define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H +#include "TableManager.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" namespace llvm { namespace jitlink { namespace aarch64 { -/// Represets aarch64 fixups enum EdgeKind_aarch64 : Edge::Kind { - - /// Set a CALL immediate field to bits [27:2] of X = Target - Fixup + Addend - R_AARCH64_CALL26 = Edge::FirstRelocation, - + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + MoveWide16, + GOTPage21, + GOTPageOffset12, + TLVPage21, + TLVPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, }; /// Returns a string name for the given aarch64 edge. For debugging purposes /// only const char *getEdgeKindName(Edge::Kind K); +// Returns whether the Instr is LD/ST (imm12) +inline bool isLoadStoreImm12(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} + +// Returns the amount the address operand of LD/ST (imm12) +// should be shifted right by. +// +// The shift value varies by the data size of LD/ST instruction. +// For instance, LDH instructoin needs the address to be shifted +// right by 1. +inline unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t Vec128Mask = 0x04800000; + + if (isLoadStoreImm12(Instr)) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +// Returns whether the Instr is MOVK/MOVZ (imm16) with a zero immediate field +inline bool isMoveWideImm16(uint32_t Instr) { + constexpr uint32_t MoveWideImm16Mask = 0x5f9fffe0; + return (Instr & MoveWideImm16Mask) == 0x52800000; +} + +// Returns the amount the address operand of MOVK/MOVZ (imm16) +// should be shifted right by. +// +// The shift value is specfied in the assembly as LSL #<shift>. +inline unsigned getMoveWide16Shift(uint32_t Instr) { + if (isMoveWideImm16(Instr)) { + uint32_t ImplicitShift = (Instr >> 21) & 0b11; + return ImplicitShift << 4; + } + + return 0; +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast<uint64_t>(Value) & 0x3) + return make_error<JITLinkError>("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits<uint32_t>::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast<uint64_t>(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (!isInt<33>(PageDelta)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error<JITLinkError>("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case MoveWide16: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue(); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(isMoveWideImm16(RawInstr) && + "RawInstr isn't a MOVK/MOVZ instruction"); + + unsigned ImmShift = getMoveWide16Shift(RawInstr); + uint32_t Imm = (TargetOffset >> ImmShift) & 0xffff; + uint32_t FixedInstr = RawInstr | (Imm << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error<JITLinkError>("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } + + return Error::success(); +} + +/// AArch64 null pointer content. +extern const uint8_t NullGOTEntryContent[8]; + +/// AArch64 PLT stub content. +extern const uint8_t StubContent[8]; + +/// Global Offset Table Builder. +class GOTTableManager : public TableManager<GOTTableManager> { +public: + static StringRef getSectionName() { return "$__GOT"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = Edge::Invalid; + const char *BlockWorkingMem = B->getContent().data(); + const char *FixupPtr = BlockWorkingMem + E.getOffset(); + + switch (E.getKind()) { + case aarch64::GOTPage21: + case aarch64::TLVPage21: { + KindToSet = aarch64::Page21; + break; + } + case aarch64::GOTPageOffset12: + case aarch64::TLVPageOffset12: { + KindToSet = aarch64::PageOffset12; + uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr; + (void)RawInstr; + assert(E.getAddend() == 0 && + "GOTPageOffset12/TLVPageOffset12 with non-zero addend"); + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + break; + } + case aarch64::PointerToGOT: { + KindToSet = aarch64::Delta64; + break; + } + default: + return false; + } + assert(KindToSet != Edge::Invalid && + "Fell through switch, but no new kind to set"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(G), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *GOTSection; + } + + ArrayRef<char> getGOTEntryBlockContent() { + return {reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; + } + + Section *GOTSection = nullptr; +}; + +/// Procedure Linkage Table Builder. +class PLTTableManager : public TableManager<PLTTableManager> { +public: + PLTTableManager(GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined()) { + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &StubContentBlock = G.createContentBlock( + getStubsSection(G), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = GOT.getEntryForTarget(G, Target); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + +public: + Section &getStubsSection(LinkGraph &G) { + if (!StubsSection) + StubsSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *StubsSection; + } + + ArrayRef<char> getStubBlockContent() { + return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)}; + } + + GOTTableManager &GOT; + Section *StubsSection = nullptr; +}; + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h b/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h index 5abd4cf11dea..95f45fae91e4 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h @@ -37,13 +37,20 @@ enum EdgeKind_riscv : Edge::Kind { /// R_RISCV_64, - /// Low 12 bits of PC-relative branch pointer value relocation + /// PC-relative branch pointer value relocation /// /// Fixup expression: - /// Fixup <- (Target - Fixup + Addend) & 0xFFF + /// Fixup <- (Target - Fixup + Addend) /// R_RISCV_BRANCH, + /// High 20 bits of PC-relative jump pointer value relocation + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend + /// + R_RISCV_JAL, + /// High 20 bits of 32-bit pointer value relocation /// /// Fixup expression @@ -145,6 +152,12 @@ enum EdgeKind_riscv : Edge::Kind { /// Fixup <- (Target - *{1}Fixup - Addend) R_RISCV_SUB8, + /// 6 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{1}Fixup - Addend) + R_RISCV_SUB6, + /// Local label assignment /// /// Fixup expression: diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h index 4a4e8d15be66..9a2bc9b09350 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -447,11 +447,10 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, break; } - default: { - // If you hit this you should check that *constructor and other non-fixup - // edges have been removed prior to applying fixups. - llvm_unreachable("Graph contains edge kind with no fixup expression"); - } + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); } return Error::success(); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index c4647148f287..df2826b50784 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -339,11 +339,7 @@ public: /// Sort the lookup set by pointer value. This sort is fast but sensitive to /// allocation order and so should not be used where a consistent order is /// required. - void sortByAddress() { - llvm::sort(Symbols, [](const value_type &LHS, const value_type &RHS) { - return LHS.first < RHS.first; - }); - } + void sortByAddress() { llvm::sort(Symbols, llvm::less_first()); } /// Sort the lookup set lexicographically. This sort is slow but the order /// is unaffected by allocation order. @@ -420,12 +416,15 @@ class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { public: static char ID; - FailedToMaterialize(std::shared_ptr<SymbolDependenceMap> Symbols); + FailedToMaterialize(std::shared_ptr<SymbolStringPool> SSP, + std::shared_ptr<SymbolDependenceMap> Symbols); + ~FailedToMaterialize(); std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; const SymbolDependenceMap &getSymbols() const { return *Symbols; } private: + std::shared_ptr<SymbolStringPool> SSP; std::shared_ptr<SymbolDependenceMap> Symbols; }; @@ -1331,7 +1330,7 @@ public: lookupInitSymbols(ExecutionSession &ES, const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms); - /// Performs an async lookup for the the given symbols in each of the given + /// Performs an async lookup for the given symbols in each of the given /// JITDylibs, calling the given handler once all lookups have completed. static void lookupInitSymbolsAsync(unique_function<void(Error)> OnComplete, @@ -1389,8 +1388,12 @@ public: /// object. ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC); + /// Destroy an ExecutionSession. Verifies that endSession was called prior to + /// destruction. + ~ExecutionSession(); + /// End the session. Closes all JITDylibs and disconnects from the - /// executor. + /// executor. Clients must call this method before destroying the session. Error endSession(); /// Get the ExecutorProcessControl object associated with this @@ -1523,7 +1526,7 @@ public: /// after resolution, the function will return a success value, but the /// error will be reported via reportErrors. Expected<SymbolMap> lookup(const JITDylibSearchOrder &SearchOrder, - const SymbolLookupSet &Symbols, + SymbolLookupSet Symbols, LookupKind K = LookupKind::Static, SymbolState RequiredState = SymbolState::Ready, RegisterDependenciesFunction RegisterDependencies = diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h index 7eb98dfc741e..c4ef06f1fbc6 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h @@ -92,6 +92,9 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); /// Render a LookupKind. raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); +/// Dump a SymbolStringPool. Useful for debugging dangling-pointer crashes. +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP); + /// A function object that can be used as an ObjectTransformLayer transform /// to dump object files to disk at a specified path. class DumpObjects { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h index 6b12fe990a8a..3804b6dda91f 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h @@ -109,7 +109,8 @@ public: /// Returns an AliasMap containing the default aliases for the ELFNixPlatform. /// This can be modified by clients when constructing the platform to add /// or remove aliases. - static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); + static Expected<SymbolAliasMap> standardPlatformAliases(ExecutionSession &ES, + JITDylib &PlatformJD); /// Returns the array of required CXX aliases. static ArrayRef<std::pair<const char *, const char *>> requiredCXXAliases(); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h index ac7051b5b75c..241453320ad5 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h @@ -23,8 +23,6 @@ #include <memory> #include <vector> -using namespace llvm::orc::shared; - namespace llvm { namespace orc { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h index 92de5882bafe..354984b540a9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h @@ -148,7 +148,7 @@ private: std::mutex EPCUIMutex; ExecutorProcessControl &EPC; std::unique_ptr<ABISupport> ABI; - JITTargetAddress ResolverBlockAddr; + JITTargetAddress ResolverBlockAddr = 0; FinalizedAlloc ResolverBlock; std::unique_ptr<TrampolinePool> TP; std::unique_ptr<LazyCallThroughManager> LCTM; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h index 2cc8c29b2813..e6a63707653a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h @@ -125,7 +125,7 @@ public: /// Set TargetOptions. /// /// Note: This operation will overwrite any previously configured options, - /// including EmulatedTLS and ExplicitEmulatedTLS which + /// including EmulatedTLS, ExplicitEmulatedTLS, and UseInitArray which /// the JITTargetMachineBuilder sets by default. Clients are responsible /// for re-enabling these overwritten options. JITTargetMachineBuilder &setOptions(TargetOptions Options) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index d76e6a21a9bb..d67a7f2bfeb2 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -56,7 +56,7 @@ public: /// Destruct this instance. If a multi-threaded instance, waits for all /// compile threads to complete. - ~LLJIT(); + virtual ~LLJIT(); /// Returns the ExecutionSession for this instance. ExecutionSession &getExecutionSession() { return *ES; } @@ -110,30 +110,30 @@ public: /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to /// look up symbols based on their IR name use the lookup function instead). - Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, - SymbolStringPtr Name); + Expected<ExecutorAddr> lookupLinkerMangled(JITDylib &JD, + SymbolStringPtr Name); /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to /// look up symbols based on their IR name use the lookup function instead). - Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, - StringRef Name) { + Expected<ExecutorAddr> lookupLinkerMangled(JITDylib &JD, + StringRef Name) { return lookupLinkerMangled(JD, ES->intern(Name)); } /// Look up a symbol in the main JITDylib by the symbol's linker-mangled name /// (to look up symbols based on their IR name use the lookup function /// instead). - Expected<JITEvaluatedSymbol> lookupLinkerMangled(StringRef Name) { + Expected<ExecutorAddr> lookupLinkerMangled(StringRef Name) { return lookupLinkerMangled(*Main, Name); } /// Look up a symbol in JITDylib JD based on its IR symbol name. - Expected<JITEvaluatedSymbol> lookup(JITDylib &JD, StringRef UnmangledName) { + Expected<ExecutorAddr> lookup(JITDylib &JD, StringRef UnmangledName) { return lookupLinkerMangled(JD, mangle(UnmangledName)); } /// Look up a symbol in the main JITDylib based on its IR symbol name. - Expected<JITEvaluatedSymbol> lookup(StringRef UnmangledName) { + Expected<ExecutorAddr> lookup(StringRef UnmangledName) { return lookup(*Main, UnmangledName); } @@ -401,7 +401,7 @@ public: std::function<std::unique_ptr<IndirectStubsManager>()>; Triple TT; - JITTargetAddress LazyCompileFailureAddr = 0; + ExecutorAddr LazyCompileFailureAddr; std::unique_ptr<LazyCallThroughManager> LCTMgr; IndirectStubsManagerBuilderFunction ISMBuilder; @@ -415,7 +415,7 @@ public: /// Set the address in the target address to call if a lazy compile fails. /// /// If this method is not called then the value will default to 0. - SetterImpl &setLazyCompileFailureAddr(JITTargetAddress Addr) { + SetterImpl &setLazyCompileFailureAddr(ExecutorAddr Addr) { this->impl().LazyCompileFailureAddr = Addr; return this->impl(); } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 01f3f1b2ab63..141dd73548c8 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -26,30 +26,19 @@ namespace llvm { namespace orc { -struct MachOJITDylibInitializers { - using SectionList = std::vector<ExecutorAddrRange>; - - MachOJITDylibInitializers(std::string Name, ExecutorAddr MachOHeaderAddress) - : Name(std::move(Name)), - MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - - std::string Name; - ExecutorAddr MachOHeaderAddress; - ExecutorAddr ObjCImageInfoAddress; - - StringMap<SectionList> InitSections; -}; - -class MachOJITDylibDeinitializers {}; - -using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>; - -using MachOJITDylibDeinitializerSequence = - std::vector<MachOJITDylibDeinitializers>; - /// Mediates between MachO initialization and ExecutionSession state. class MachOPlatform : public Platform { public: + // Used internally by MachOPlatform, but made public to enable serialization. + struct MachOJITDylibDepInfo { + bool Sealed = false; + std::vector<ExecutorAddr> DepHeaders; + }; + + // Used internally by MachOPlatform, but made public to enable serialization. + using MachOJITDylibDepInfoMap = + std::vector<std::pair<ExecutorAddr, MachOJITDylibDepInfo>>; + /// Try to create a MachOPlatform instance, adding the ORC runtime to the /// given JITDylib. /// @@ -161,26 +150,28 @@ private: Error processObjCImageInfo(jitlink::LinkGraph &G, MaterializationResponsibility &MR); - Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); - Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD); - Error registerEHAndTLVSections(jitlink::LinkGraph &G); + Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD); Error registerEHSectionsPhase1(jitlink::LinkGraph &G); std::mutex PluginMutex; MachOPlatform &MP; + + // FIXME: ObjCImageInfos and HeaderAddrs need to be cleared when + // JITDylibs are removed. DenseMap<JITDylib *, std::pair<uint32_t, uint32_t>> ObjCImageInfos; + DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs; InitSymbolDepMap InitSymbolDeps; }; - using SendInitializerSequenceFn = - unique_function<void(Expected<MachOJITDylibInitializerSequence>)>; - - using SendDeinitializerSequenceFn = - unique_function<void(Expected<MachOJITDylibDeinitializerSequence>)>; - + using GetJITDylibHeaderSendResultFn = + unique_function<void(Expected<ExecutorAddr>)>; + using GetJITDylibNameSendResultFn = + unique_function<void(Expected<StringRef>)>; + using PushInitializersSendResultFn = + unique_function<void(Expected<MachOJITDylibDepInfoMap>)>; using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddr>)>; static bool supportedTarget(const Triple &TT); @@ -193,28 +184,24 @@ private: // Associate MachOPlatform JIT-side runtime support functions with handlers. Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); - void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, - JITDylib &JD, - std::vector<JITDylibSP> DFSLinkOrder); + // Implements rt_pushInitializers by making repeat async lookups for + // initializer symbols (each lookup may spawn more initializer symbols if + // it pulls in new materializers, e.g. from objects in a static library). + void pushInitializersLoop(PushInitializersSendResultFn SendResult, + JITDylibSP JD); - void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, - JITDylib &JD); - - void rt_getInitializers(SendInitializerSequenceFn SendResult, - StringRef JDName); - - void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, - ExecutorAddr Handle); + // Handle requests from the ORC runtime to push MachO initializer info. + void rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr); + // Handle requests for symbol addresses from the ORC runtime. void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName); // Records the addresses of runtime symbols used by the platform. Error bootstrapMachORuntime(JITDylib &PlatformJD); - Error registerInitInfo(JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, - ArrayRef<jitlink::Section *> InitSections); - + // Call the ORC runtime to create a pthread key. Expected<uint64_t> createPThreadKey(); enum PlatformState { BootstrapPhase1, BootstrapPhase2, Initialized }; @@ -229,81 +216,24 @@ private: ExecutorAddr orc_rt_macho_platform_shutdown; ExecutorAddr orc_rt_macho_register_ehframe_section; ExecutorAddr orc_rt_macho_deregister_ehframe_section; - ExecutorAddr orc_rt_macho_register_thread_data_section; - ExecutorAddr orc_rt_macho_deregister_thread_data_section; + ExecutorAddr orc_rt_macho_register_jitdylib; + ExecutorAddr orc_rt_macho_deregister_jitdylib; + ExecutorAddr orc_rt_macho_register_object_platform_sections; + ExecutorAddr orc_rt_macho_deregister_object_platform_sections; ExecutorAddr orc_rt_macho_create_pthread_key; DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols; - // InitSeqs gets its own mutex to avoid locking the whole session when - // aggregating data from the jitlink. std::mutex PlatformMutex; - DenseMap<JITDylib *, MachOJITDylibInitializers> InitSeqs; - + DenseMap<JITDylib *, ExecutorAddr> JITDylibToHeaderAddr; DenseMap<ExecutorAddr, JITDylib *> HeaderAddrToJITDylib; DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey; }; namespace shared { -using SPSNamedExecutorAddrRangeSequenceMap = - SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>; - -using SPSMachOJITDylibInitializers = - SPSTuple<SPSString, SPSExecutorAddr, SPSExecutorAddr, - SPSNamedExecutorAddrRangeSequenceMap>; - -using SPSMachOJITDylibInitializerSequence = - SPSSequence<SPSMachOJITDylibInitializers>; - -/// Serialization traits for MachOJITDylibInitializers. -template <> -class SPSSerializationTraits<SPSMachOJITDylibInitializers, - MachOJITDylibInitializers> { -public: - static size_t size(const MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::size( - MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool serialize(SPSOutputBuffer &OB, - const MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::serialize( - OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool deserialize(SPSInputBuffer &IB, - MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::deserialize( - IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } -}; - -using SPSMachOJITDylibDeinitializers = SPSEmpty; - -using SPSMachOJITDylibDeinitializerSequence = - SPSSequence<SPSMachOJITDylibDeinitializers>; - -template <> -class SPSSerializationTraits<SPSMachOJITDylibDeinitializers, - MachOJITDylibDeinitializers> { -public: - static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } - - static bool serialize(SPSOutputBuffer &OB, - const MachOJITDylibDeinitializers &MOJDDs) { - return true; - } - - static bool deserialize(SPSInputBuffer &IB, - MachOJITDylibDeinitializers &MOJDDs) { - MOJDDs = MachOJITDylibDeinitializers(); - return true; - } -}; +using SPSNamedExecutorAddrRangeSequence = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; } // end namespace shared } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h new file mode 100644 index 000000000000..d023bfbdb5b6 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h @@ -0,0 +1,115 @@ +//===- MemoryMapper.h - Cross-process memory mapper -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Cross-process (and in-process) memory mapping and transfer +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H +#define LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H + +#include "llvm/ExecutionEngine/Orc/Core.h" + +#include <mutex> + +namespace llvm { +namespace orc { + +/// Manages mapping, content transfer and protections for JIT memory +class MemoryMapper { +public: + /// Represents a single allocation containing multiple segments and + /// initialization and deinitialization actions + struct AllocInfo { + struct SegInfo { + ExecutorAddrDiff Offset; + const char *WorkingMem; + size_t ContentSize; + size_t ZeroFillSize; + unsigned Prot; + }; + + ExecutorAddr MappingBase; + std::vector<SegInfo> Segments; + shared::AllocActions Actions; + }; + + using OnReservedFunction = unique_function<void(Expected<ExecutorAddrRange>)>; + + /// Reserves address space in executor process + virtual void reserve(size_t NumBytes, OnReservedFunction OnReserved) = 0; + + /// Provides working memory + virtual char *prepare(ExecutorAddr Addr, size_t ContentSize) = 0; + + using OnInitializedFunction = unique_function<void(Expected<ExecutorAddr>)>; + + /// Ensures executor memory is synchronized with working copy memory, sends + /// functions to be called after initilization and before deinitialization and + /// applies memory protections + /// Returns a unique address identifying the allocation. This address should + /// be passed to deinitialize to run deallocation actions (and reset + /// permissions where possible). + virtual void initialize(AllocInfo &AI, + OnInitializedFunction OnInitialized) = 0; + + using OnDeinitializedFunction = unique_function<void(Error)>; + + /// Runs previously specified deinitialization actions + /// Executor addresses returned by initialize should be passed + virtual void deinitialize(ArrayRef<ExecutorAddr> Allocations, + OnDeinitializedFunction OnDeInitialized) = 0; + + using OnReleasedFunction = unique_function<void(Error)>; + + /// Release address space acquired through reserve() + virtual void release(ArrayRef<ExecutorAddr> Reservations, + OnReleasedFunction OnRelease) = 0; + + virtual ~MemoryMapper(); +}; + +class InProcessMemoryMapper final : public MemoryMapper { +public: + InProcessMemoryMapper() {} + + void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; + + void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override; + + char *prepare(ExecutorAddr Addr, size_t ContentSize) override; + + void deinitialize(ArrayRef<ExecutorAddr> Allocations, + OnDeinitializedFunction OnDeInitialized) override; + + void release(ArrayRef<ExecutorAddr> Reservations, + OnReleasedFunction OnRelease) override; + + ~InProcessMemoryMapper() override; + +private: + struct Allocation { + std::vector<shared::WrapperFunctionCall> DeinitializationActions; + }; + using AllocationMap = DenseMap<ExecutorAddr, Allocation>; + + struct Reservation { + size_t Size; + std::vector<ExecutorAddr> Allocations; + }; + using ReservationMap = DenseMap<void *, Reservation>; + + std::mutex Mutex; + ReservationMap Reservations; + AllocationMap Allocations; +}; + +} // namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h index 82dfdc270128..c5c2780bc9ee 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -330,6 +330,45 @@ public: JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); }; +// @brief riscv64 support. +// +// RISC-V 64 supports lazy JITing. +class OrcRiscv64 { +public: + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 16; + static constexpr unsigned StubSize = 16; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0x148; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverFnAddr, + unsigned NumTrampolines); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); +}; + } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h index dc080cfc79d1..5d545f8abdb9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h @@ -43,13 +43,22 @@ public: /// Cast this ExecutorAddr to a pointer of the given type. /// Warning: This should only be used when JITing in-process. - template <typename T> T toPtr() const { - static_assert(std::is_pointer<T>::value, "T must be a pointer type"); + template <typename T> + std::enable_if_t<std::is_pointer<T>::value, T> toPtr() const { uintptr_t IntPtr = static_cast<uintptr_t>(Addr); assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t"); return reinterpret_cast<T>(IntPtr); } + /// Cast this ExecutorAddr to a pointer of the given function type. + /// Warning: This should only be used when JITing in-process. + template <typename T> + std::enable_if_t<std::is_function<T>::value, T *> toPtr() const { + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t"); + return reinterpret_cast<T *>(IntPtr); + } + uint64_t getValue() const { return Addr; } void setValue(uint64_t Addr) { this->Addr = Addr; } bool isNull() const { return Addr == 0; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index 302b60b80fd0..9be58e9f0fa9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -586,7 +586,7 @@ SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) { if (E) return {true, std::move(*E), {}}; else - return {false, {}, toString(E.takeError())}; + return {false, T(), toString(E.takeError())}; } template <typename T> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h index a138f60a7756..b7bba7a48786 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -88,7 +88,7 @@ private: for (auto &Callee : CandidateSet) { auto ImplSymbol = AliaseeImplTable.getImplFor(Callee); // try to distinguish already compiled & library symbols - if (!ImplSymbol.hasValue()) + if (!ImplSymbol) continue; const auto &ImplSymbolName = ImplSymbol.getPointer()->first; JITDylib *ImplJD = ImplSymbol.getPointer()->second; @@ -175,9 +175,8 @@ public: using ResultEval = std::function<IRlikiesStrRef(Function &)>; using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>; - IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, - Speculator &Spec, MangleAndInterner &Mangle, - ResultEval Interpreter) + IRSpeculationLayer(ExecutionSession &ES, IRLayer &BaseLayer, Speculator &Spec, + MangleAndInterner &Mangle, ResultEval Interpreter) : IRLayer(ES, BaseLayer.getManglingOptions()), NextLayer(BaseLayer), S(Spec), Mangle(Mangle), QueryAnalysis(Interpreter) {} @@ -198,7 +197,7 @@ private: return InternedNames; } - IRCompileLayer &NextLayer; + IRLayer &NextLayer; Speculator &S; MangleAndInterner &Mangle; ResultEval QueryAnalysis; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index 63abb196ba49..7e433965c922 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -19,6 +19,9 @@ #include <mutex> namespace llvm { + +class raw_ostream; + namespace orc { class SymbolStringPtr; @@ -26,6 +29,10 @@ class SymbolStringPtr; /// String pool for symbol names used by the JIT. class SymbolStringPool { friend class SymbolStringPtr; + + // Implemented in DebugUtils.h. + friend raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP); + public: /// Destroy a SymbolStringPool. ~SymbolStringPool(); diff --git a/llvm/include/llvm/FileCheck/FileCheck.h b/llvm/include/llvm/FileCheck/FileCheck.h index 7a6c98db3029..d6d8dc531e10 100644 --- a/llvm/include/llvm/FileCheck/FileCheck.h +++ b/llvm/include/llvm/FileCheck/FileCheck.h @@ -14,14 +14,17 @@ #define LLVM_FILECHECK_FILECHECK_H #include "llvm/ADT/StringRef.h" -#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" -#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/SMLoc.h" #include <bitset> +#include <memory> #include <string> #include <vector> namespace llvm { +class MemoryBuffer; +class SourceMgr; +template <typename T> class SmallVectorImpl; /// Contains info about various FileCheck options. struct FileCheckRequest { @@ -45,6 +48,7 @@ namespace Check { enum FileCheckKind { CheckNone = 0, + CheckMisspelled, CheckPlain, CheckNext, CheckSame, diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index c5abb16dd9e5..5f1d335ef04f 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -122,13 +122,12 @@ def OMPC_ProcBind : Clause<"proc_bind"> { ]; } -// static and auto are C++ keywords so need a capital to disambiguate. -def OMP_SCHEDULE_Static : ClauseVal<"Static", 2, 1> {} -def OMP_SCHEDULE_Dynamic : ClauseVal<"Dynamic", 3, 1> {} -def OMP_SCHEDULE_Guided : ClauseVal<"Guided", 4, 1> {} -def OMP_SCHEDULE_Auto : ClauseVal<"Auto", 5, 1> {} -def OMP_SCHEDULE_Runtime : ClauseVal<"Runtime", 6, 1> {} -def OMP_SCHEDULE_Default : ClauseVal<"Default", 7, 0> { let isDefault = 1; } +def OMP_SCHEDULE_Static : ClauseVal<"static", 2, 1> {} +def OMP_SCHEDULE_Dynamic : ClauseVal<"dynamic", 3, 1> {} +def OMP_SCHEDULE_Guided : ClauseVal<"guided", 4, 1> {} +def OMP_SCHEDULE_Auto : ClauseVal<"auto", 5, 1> {} +def OMP_SCHEDULE_Runtime : ClauseVal<"runtime", 6, 1> {} +def OMP_SCHEDULE_Default : ClauseVal<"default", 7, 0> { let isDefault = 1; } def OMPC_Schedule : Clause<"schedule"> { let clangClass = "OMPScheduleClause"; @@ -164,6 +163,25 @@ def OMPC_MemoryOrder : Clause<"memory_order"> { ]; } +def OMP_CANCELLATION_CONSTRUCT_Parallel : ClauseVal<"parallel", 1, 1> {} +def OMP_CANCELLATION_CONSTRUCT_Loop : ClauseVal<"loop", 2, 1> {} +def OMP_CANCELLATION_CONSTRUCT_Sections : ClauseVal<"sections", 3, 1> {} +def OMP_CANCELLATION_CONSTRUCT_Taskgroup : ClauseVal<"taskgroup", 4, 1> {} +def OMP_CANCELLATION_CONSTRUCT_None : ClauseVal<"none", 5, 0> { + let isDefault = 1; +} + +def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> { + let enumClauseValue = "CancellationConstructType"; + let allowedClauseValues = [ + OMP_CANCELLATION_CONSTRUCT_Parallel, + OMP_CANCELLATION_CONSTRUCT_Loop, + OMP_CANCELLATION_CONSTRUCT_Sections, + OMP_CANCELLATION_CONSTRUCT_Taskgroup, + OMP_CANCELLATION_CONSTRUCT_None + ]; +} + def OMPC_Ordered : Clause<"ordered"> { let clangClass = "OMPOrderedClause"; let flangClass = "ScalarIntConstantExpr"; @@ -254,12 +272,18 @@ def OMPC_IsDevicePtr : Clause<"is_device_ptr"> { let flangClass = "Name"; let isValueList = true; } +def OMPC_HasDeviceAddr : Clause<"has_device_addr"> { + let clangClass = "OMPHasDeviceAddrClause"; + let flangClass = "Name"; + let isValueList = true; +} def OMPC_TaskReduction : Clause<"task_reduction"> { let clangClass = "OMPTaskReductionClause"; let flangClass = "OmpReductionClause"; } def OMPC_InReduction : Clause<"in_reduction"> { let clangClass = "OMPInReductionClause"; + let flangClass = "OmpInReductionClause"; } def OMPC_UnifiedAddress : Clause<"unified_address"> { let clangClass = "OMPUnifiedAddressClause"; @@ -557,7 +581,9 @@ def OMP_Target : Directive<"target"> { VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_InReduction, 50>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators, 50> ]; @@ -590,11 +616,20 @@ def OMP_Requires : Directive<"requires"> { let allowedClauses = [ VersionedClause<OMPC_UnifiedAddress>, VersionedClause<OMPC_UnifiedSharedMemory>, - VersionedClause<OMPC_ReverseOffload>, + // OpenMP 5.2 Spec: If an implementation is not supporting a requirement + // (reverse offload in this case) then it should give compile-time error + // termination. + // Seeting supported version for reverse_offload to a distant future version + // 9.9 so that its partial support can be tested in the meantime. + // + // TODO: Correct this supprted version number whenever complete + // implementation of reverse_offload is available. + VersionedClause<OMPC_ReverseOffload, 99>, VersionedClause<OMPC_DynamicAllocators>, VersionedClause<OMPC_AtomicDefaultMemOrder> ]; } +def OMP_Nothing : Directive<"nothing"> {} def OMP_TargetData : Directive<"target data"> { let allowedClauses = [ VersionedClause<OMPC_UseDevicePtr>, @@ -645,6 +680,7 @@ def OMP_TargetParallel : Directive<"target parallel"> { VersionedClause<OMPC_Shared>, VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators, 50> ]; @@ -677,6 +713,7 @@ def OMP_TargetParallelFor : Directive<"target parallel for"> { VersionedClause<OMPC_Ordered>, VersionedClause<OMPC_Linear>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_Order, 50>, VersionedClause<OMPC_UsesAllocators, 50> @@ -693,6 +730,7 @@ def OMP_TargetParallelDo : Directive<"target parallel do"> { VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_Linear>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Allocator>, VersionedClause<OMPC_Order>, VersionedClause<OMPC_UsesAllocators>, @@ -825,6 +863,21 @@ def OMP_ParallelMaster : Directive<"parallel master"> { VersionedClause<OMPC_Allocate> ]; } +def OMP_ParallelMasked : Directive<"parallel masked"> { + let allowedClauses = [ + VersionedClause<OMPC_If>, + VersionedClause<OMPC_NumThreads>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_Copyin>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_ProcBind>, + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_Filter> + ]; +} def OMP_ParallelSections : Directive<"parallel sections"> { let allowedClauses = [ VersionedClause<OMPC_If>, @@ -1126,6 +1179,7 @@ def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> { VersionedClause<OMPC_SimdLen>, VersionedClause<OMPC_Aligned>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_NonTemporal, 50>, VersionedClause<OMPC_Order, 50>, @@ -1156,6 +1210,7 @@ def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> { VersionedClause<OMPC_SimdLen>, VersionedClause<OMPC_Aligned>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_NonTemporal>, VersionedClause<OMPC_Order>, @@ -1169,6 +1224,7 @@ def OMP_TargetSimd : Directive<"target simd"> { VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_LastPrivate>, VersionedClause<OMPC_Linear>, VersionedClause<OMPC_Map>, @@ -1342,6 +1398,7 @@ def OMP_TargetTeams : Directive<"target teams"> { VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators, 50>, @@ -1365,6 +1422,7 @@ def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> { VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators, 50>, @@ -1395,6 +1453,7 @@ def OMP_TargetTeamsDistributeParallelFor : VersionedClause<OMPC_DefaultMap>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Default>, VersionedClause<OMPC_Shared>, VersionedClause<OMPC_Reduction>, @@ -1420,6 +1479,7 @@ def OMP_TargetTeamsDistributeParallelDo : VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators>, @@ -1456,6 +1516,7 @@ def OMP_TargetTeamsDistributeParallelForSimd : VersionedClause<OMPC_DefaultMap>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Default>, VersionedClause<OMPC_Shared>, VersionedClause<OMPC_Reduction>, @@ -1485,6 +1546,7 @@ def OMP_TargetTeamsDistributeParallelDoSimd : VersionedClause<OMPC_Depend>, VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_Reduction>, VersionedClause<OMPC_Allocate>, VersionedClause<OMPC_UsesAllocators>, @@ -1523,6 +1585,7 @@ def OMP_TargetTeamsDistributeSimd : VersionedClause<OMPC_FirstPrivate>, VersionedClause<OMPC_If>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_LastPrivate>, VersionedClause<OMPC_Linear>, VersionedClause<OMPC_Map>, @@ -1581,6 +1644,28 @@ def OMP_MasterTaskloop : Directive<"master taskloop"> { VersionedClause<OMPC_Allocate> ]; } +def OMP_MaskedTaskloop : Directive<"masked taskloop"> { + let allowedClauses = [ + VersionedClause<OMPC_If>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Final>, + VersionedClause<OMPC_Untied>, + VersionedClause<OMPC_Mergeable>, + VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_GrainSize>, + VersionedClause<OMPC_NoGroup>, + VersionedClause<OMPC_NumTasks>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_InReduction>, + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_Filter> + ]; +} def OMP_ParallelMasterTaskloop : Directive<"parallel master taskloop"> { let allowedClauses = [ @@ -1605,6 +1690,31 @@ def OMP_ParallelMasterTaskloop : VersionedClause<OMPC_Copyin> ]; } +def OMP_ParallelMaskedTaskloop : + Directive<"parallel masked taskloop"> { + let allowedClauses = [ + VersionedClause<OMPC_If>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Final>, + VersionedClause<OMPC_Untied>, + VersionedClause<OMPC_Mergeable>, + VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_GrainSize>, + VersionedClause<OMPC_NoGroup>, + VersionedClause<OMPC_NumTasks>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_NumThreads>, + VersionedClause<OMPC_ProcBind>, + VersionedClause<OMPC_Copyin>, + VersionedClause<OMPC_Filter> + ]; +} def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { let allowedClauses = [ VersionedClause<OMPC_If>, @@ -1632,6 +1742,34 @@ def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { VersionedClause<OMPC_Order, 50> ]; } +def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> { + let allowedClauses = [ + VersionedClause<OMPC_If>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Final>, + VersionedClause<OMPC_Untied>, + VersionedClause<OMPC_Mergeable>, + VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_Linear>, + VersionedClause<OMPC_Aligned>, + VersionedClause<OMPC_SafeLen>, + VersionedClause<OMPC_SimdLen>, + VersionedClause<OMPC_GrainSize>, + VersionedClause<OMPC_NoGroup>, + VersionedClause<OMPC_NumTasks>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_InReduction>, + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_NonTemporal, 50>, + VersionedClause<OMPC_Order, 50>, + VersionedClause<OMPC_Filter> + ]; +} def OMP_ParallelMasterTaskloopSimd : Directive<"parallel master taskloop simd"> { let allowedClauses = [ @@ -1662,6 +1800,37 @@ def OMP_ParallelMasterTaskloopSimd : VersionedClause<OMPC_Order, 50> ]; } +def OMP_ParallelMaskedTaskloopSimd : + Directive<"parallel masked taskloop simd"> { + let allowedClauses = [ + VersionedClause<OMPC_If>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Final>, + VersionedClause<OMPC_Untied>, + VersionedClause<OMPC_Mergeable>, + VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_GrainSize>, + VersionedClause<OMPC_NoGroup>, + VersionedClause<OMPC_NumTasks>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_NumThreads>, + VersionedClause<OMPC_ProcBind>, + VersionedClause<OMPC_Copyin>, + VersionedClause<OMPC_Linear>, + VersionedClause<OMPC_Aligned>, + VersionedClause<OMPC_SafeLen>, + VersionedClause<OMPC_SimdLen>, + VersionedClause<OMPC_NonTemporal, 50>, + VersionedClause<OMPC_Order, 50>, + VersionedClause<OMPC_Filter> + ]; +} def OMP_Depobj : Directive<"depobj"> { let allowedClauses = [ VersionedClause<OMPC_Depend, 50>, @@ -1734,6 +1903,7 @@ def OMP_dispatch : Directive<"dispatch"> { let allowedClauses = [ VersionedClause<OMPC_Device>, VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, VersionedClause<OMPC_NoWait>, VersionedClause<OMPC_Depend>, VersionedClause<OMPC_Novariants>, @@ -1757,6 +1927,99 @@ def OMP_loop : Directive<"loop"> { VersionedClause<OMPC_Order>, ]; } +def OMP_teams_loop : Directive<"teams loop"> { + let allowedClauses = [ + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Shared>, + ]; + let allowedOnceClauses = [ + VersionedClause<OMPC_Bind, 50>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_NumTeams>, + VersionedClause<OMPC_Order>, + VersionedClause<OMPC_ThreadLimit>, + ]; +} +def OMP_target_teams_loop : Directive<"target teams loop"> { + let allowedClauses = [ + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_Depend>, + VersionedClause<OMPC_DefaultMap>, + VersionedClause<OMPC_Device>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Map>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_UsesAllocators, 50> + ]; + let allowedOnceClauses = [ + VersionedClause<OMPC_Bind, 50>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_If>, + VersionedClause<OMPC_NoWait>, + VersionedClause<OMPC_NumTeams>, + VersionedClause<OMPC_Order>, + VersionedClause<OMPC_ThreadLimit>, + ]; +} +def OMP_parallel_loop : Directive<"parallel loop"> { + let allowedClauses = [ + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_Copyin>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Shared>, + ]; + let allowedOnceClauses = [ + VersionedClause<OMPC_Bind, 50>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_If>, + VersionedClause<OMPC_NumThreads>, + VersionedClause<OMPC_Order>, + VersionedClause<OMPC_ProcBind>, + ]; +} +def OMP_target_parallel_loop : Directive<"target parallel loop"> { + let allowedClauses = [ + VersionedClause<OMPC_Allocate>, + VersionedClause<OMPC_Copyin>, + VersionedClause<OMPC_Depend>, + VersionedClause<OMPC_Device>, + VersionedClause<OMPC_FirstPrivate>, + VersionedClause<OMPC_IsDevicePtr>, + VersionedClause<OMPC_HasDeviceAddr, 51>, + VersionedClause<OMPC_LastPrivate>, + VersionedClause<OMPC_Map>, + VersionedClause<OMPC_Private>, + VersionedClause<OMPC_Reduction>, + VersionedClause<OMPC_Shared>, + VersionedClause<OMPC_UsesAllocators, 50>, + ]; + let allowedOnceClauses = [ + VersionedClause<OMPC_Bind, 50>, + VersionedClause<OMPC_Collapse>, + VersionedClause<OMPC_Default>, + VersionedClause<OMPC_DefaultMap>, + VersionedClause<OMPC_If>, + VersionedClause<OMPC_NoWait>, + VersionedClause<OMPC_NumThreads>, + VersionedClause<OMPC_Order>, + VersionedClause<OMPC_ProcBind>, + ]; +} def OMP_Metadirective : Directive<"metadirective"> { let allowedClauses = [VersionedClause<OMPC_When>]; let allowedOnceClauses = [VersionedClause<OMPC_Default>]; diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h index bee90281e086..76104f6bc9cf 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -74,26 +74,114 @@ enum class IdentFlag { /// \note This needs to be kept in sync with kmp.h enum sched_type. /// Todo: Update kmp.h to include this file, and remove the enums in kmp.h -/// To complete this, more enum values will need to be moved here. enum class OMPScheduleType { - StaticChunked = 33, - Static = 34, // static unspecialized - DistributeChunked = 91, - Distribute = 92, - DynamicChunked = 35, - GuidedChunked = 36, // guided unspecialized - Runtime = 37, - Auto = 38, // auto - - StaticBalancedChunked = 45, // static with chunk adjustment (e.g., simd) - GuidedSimd = 46, // guided with chunk adjustment - RuntimeSimd = 47, // runtime with chunk adjustment - - ModifierMonotonic = - (1 << 29), // Set if the monotonic schedule modifier was present - ModifierNonmonotonic = - (1 << 30), // Set if the nonmonotonic schedule modifier was present - ModifierMask = ModifierMonotonic | ModifierNonmonotonic, + // For typed comparisons, not a valid schedule + None = 0, + + // Schedule algorithms + BaseStaticChunked = 1, + BaseStatic = 2, + BaseDynamicChunked = 3, + BaseGuidedChunked = 4, + BaseRuntime = 5, + BaseAuto = 6, + BaseTrapezoidal = 7, + BaseGreedy = 8, + BaseBalanced = 9, + BaseGuidedIterativeChunked = 10, + BaseGuidedAnalyticalChunked = 11, + BaseSteal = 12, + + // with chunk adjustment (e.g., simd) + BaseStaticBalancedChunked = 13, + BaseGuidedSimd = 14, + BaseRuntimeSimd = 15, + + // static schedules algorithims for distribute + BaseDistributeChunked = 27, + BaseDistribute = 28, + + // Modifier flags to be combined with schedule algorithms + ModifierUnordered = (1 << 5), + ModifierOrdered = (1 << 6), + ModifierNomerge = (1 << 7), + ModifierMonotonic = (1 << 29), + ModifierNonmonotonic = (1 << 30), + + // Masks combining multiple flags + OrderingMask = ModifierUnordered | ModifierOrdered | ModifierNomerge, + MonotonicityMask = ModifierMonotonic | ModifierNonmonotonic, + ModifierMask = OrderingMask | MonotonicityMask, + + // valid schedule type values, without monotonicity flags + UnorderedStaticChunked = BaseStaticChunked | ModifierUnordered, // 33 + UnorderedStatic = BaseStatic | ModifierUnordered, // 34 + UnorderedDynamicChunked = BaseDynamicChunked | ModifierUnordered, // 35 + UnorderedGuidedChunked = BaseGuidedChunked | ModifierUnordered, // 36 + UnorderedRuntime = BaseRuntime | ModifierUnordered, // 37 + UnorderedAuto = BaseAuto | ModifierUnordered, // 38 + UnorderedTrapezoidal = BaseTrapezoidal | ModifierUnordered, // 39 + UnorderedGreedy = BaseGreedy | ModifierUnordered, // 40 + UnorderedBalanced = BaseBalanced | ModifierUnordered, // 41 + UnorderedGuidedIterativeChunked = + BaseGuidedIterativeChunked | ModifierUnordered, // 42 + UnorderedGuidedAnalyticalChunked = + BaseGuidedAnalyticalChunked | ModifierUnordered, // 43 + UnorderedSteal = BaseSteal | ModifierUnordered, // 44 + + UnorderedStaticBalancedChunked = + BaseStaticBalancedChunked | ModifierUnordered, // 45 + UnorderedGuidedSimd = BaseGuidedSimd | ModifierUnordered, // 46 + UnorderedRuntimeSimd = BaseRuntimeSimd | ModifierUnordered, // 47 + + OrderedStaticChunked = BaseStaticChunked | ModifierOrdered, // 65 + OrderedStatic = BaseStatic | ModifierOrdered, // 66 + OrderedDynamicChunked = BaseDynamicChunked | ModifierOrdered, // 67 + OrderedGuidedChunked = BaseGuidedChunked | ModifierOrdered, // 68 + OrderedRuntime = BaseRuntime | ModifierOrdered, // 69 + OrderedAuto = BaseAuto | ModifierOrdered, // 70 + OrderdTrapezoidal = BaseTrapezoidal | ModifierOrdered, // 71 + + OrderedDistributeChunked = BaseDistributeChunked | ModifierOrdered, // 91 + OrderedDistribute = BaseDistribute | ModifierOrdered, // 92 + + NomergeUnorderedStaticChunked = + BaseStaticChunked | ModifierUnordered | ModifierNomerge, // 161 + NomergeUnorderedStatic = + BaseStatic | ModifierUnordered | ModifierNomerge, // 162 + NomergeUnorderedDynamicChunked = + BaseDynamicChunked | ModifierUnordered | ModifierNomerge, // 163 + NomergeUnorderedGuidedChunked = + BaseGuidedChunked | ModifierUnordered | ModifierNomerge, // 164 + NomergeUnorderedRuntime = + BaseRuntime | ModifierUnordered | ModifierNomerge, // 165 + NomergeUnorderedAuto = BaseAuto | ModifierUnordered | ModifierNomerge, // 166 + NomergeUnorderedTrapezoidal = + BaseTrapezoidal | ModifierUnordered | ModifierNomerge, // 167 + NomergeUnorderedGreedy = + BaseGreedy | ModifierUnordered | ModifierNomerge, // 168 + NomergeUnorderedBalanced = + BaseBalanced | ModifierUnordered | ModifierNomerge, // 169 + NomergeUnorderedGuidedIterativeChunked = + BaseGuidedIterativeChunked | ModifierUnordered | ModifierNomerge, // 170 + NomergeUnorderedGuidedAnalyticalChunked = + BaseGuidedAnalyticalChunked | ModifierUnordered | ModifierNomerge, // 171 + NomergeUnorderedSteal = + BaseSteal | ModifierUnordered | ModifierNomerge, // 172 + + NomergeOrderedStaticChunked = + BaseStaticChunked | ModifierOrdered | ModifierNomerge, // 193 + NomergeOrderedStatic = BaseStatic | ModifierOrdered | ModifierNomerge, // 194 + NomergeOrderedDynamicChunked = + BaseDynamicChunked | ModifierOrdered | ModifierNomerge, // 195 + NomergeOrderedGuidedChunked = + BaseGuidedChunked | ModifierOrdered | ModifierNomerge, // 196 + NomergeOrderedRuntime = + BaseRuntime | ModifierOrdered | ModifierNomerge, // 197 + NomergeOrderedAuto = BaseAuto | ModifierOrdered | ModifierNomerge, // 198 + NomergeOrderedTrapezoidal = + BaseTrapezoidal | ModifierOrdered | ModifierNomerge, // 199 + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue */ ModifierMask) }; @@ -116,6 +204,9 @@ enum class AddressSpace : unsigned { /// \note This needs to be kept in sync with interop.h enum kmp_interop_type_t.: enum class OMPInteropType { Unknown, Target, TargetSync }; +/// Atomic compare operations. Currently OpenMP only supports ==, >, and <. +enum class OMPAtomicCompareOp : unsigned { EQ, MIN, MAX }; + } // end namespace omp } // end namespace llvm diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h index 544f698655a4..b13b74ceab86 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h @@ -15,14 +15,14 @@ #ifndef LLVM_FRONTEND_OPENMP_OMPCONTEXT_H #define LLVM_FRONTEND_OPENMP_OMPCONTEXT_H -#include "llvm/ADT/APSInt.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/BitVector.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Triple.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/Frontend/OpenMP/OMPConstants.h" namespace llvm { +class Triple; namespace omp { /// OpenMP Context related IDs and helpers diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index f60debe8411c..8a6b1c7d412d 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -23,6 +23,52 @@ namespace llvm { class CanonicalLoopInfo; +/// Move the instruction after an InsertPoint to the beginning of another +/// BasicBlock. +/// +/// The instructions after \p IP are moved to the beginning of \p New which must +/// not have any PHINodes. If \p CreateBranch is true, a branch instruction to +/// \p New will be added such that there is no semantic change. Otherwise, the +/// \p IP insert block remains degenerate and it is up to the caller to insert a +/// terminator. +void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New, + bool CreateBranch); + +/// Splice a BasicBlock at an IRBuilder's current insertion point. Its new +/// insert location will stick to after the instruction before the insertion +/// point (instead of moving with the instruction the InsertPoint stores +/// internally). +void spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch); + +/// Split a BasicBlock at an InsertPoint, even if the block is degenerate +/// (missing the terminator). +/// +/// llvm::SplitBasicBlock and BasicBlock::splitBasicBlock require a well-formed +/// BasicBlock. \p Name is used for the new successor block. If \p CreateBranch +/// is true, a branch to the new successor will new created such that +/// semantically there is no change; otherwise the block of the insertion point +/// remains degenerate and it is the caller's responsibility to insert a +/// terminator. Returns the new successor block. +BasicBlock *splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch, + llvm::Twine Name = {}); + +/// Split a BasicBlock at \p Builder's insertion point, even if the block is +/// degenerate (missing the terminator). Its new insert location will stick to +/// after the instruction before the insertion point (instead of moving with the +/// instruction the InsertPoint stores internally). +BasicBlock *splitBB(IRBuilderBase &Builder, bool CreateBranch, + llvm::Twine Name = {}); + +/// Split a BasicBlock at \p Builder's insertion point, even if the block is +/// degenerate (missing the terminator). Its new insert location will stick to +/// after the instruction before the insertion point (instead of moving with the +/// instruction the InsertPoint stores internally). +BasicBlock *splitBB(IRBuilder<> &Builder, bool CreateBranch, llvm::Twine Name); + +/// Like splitBB, but reuses the current block's name for the new name. +BasicBlock *splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch, + llvm::Twine Suffix = ".split"); + /// An interface to create LLVM-IR for OpenMP directives. /// /// Each OpenMP directive has a corresponding public generator method. @@ -87,27 +133,36 @@ public: /// Callback type for body (=inner region) code generation /// /// The callback takes code locations as arguments, each describing a - /// location at which code might need to be generated or a location that is - /// the target of control transfer. + /// location where additional instructions can be inserted. + /// + /// The CodeGenIP may be in the middle of a basic block or point to the end of + /// it. The basic block may have a terminator or be degenerate. The callback + /// function may just insert instructions at that position, but also split the + /// block (without the Before argument of BasicBlock::splitBasicBlock such + /// that the identify of the split predecessor block is preserved) and insert + /// additional control flow, including branches that do not lead back to what + /// follows the CodeGenIP. Note that since the callback is allowed to split + /// the block, callers must assume that InsertPoints to positions in the + /// BasicBlock after CodeGenIP including CodeGenIP itself are invalidated. If + /// such InsertPoints need to be preserved, it can split the block itself + /// before calling the callback. + /// + /// AllocaIP and CodeGenIP must not point to the same position. /// /// \param AllocaIP is the insertion point at which new alloca instructions - /// should be placed. + /// should be placed. The BasicBlock it is pointing to must + /// not be split. /// \param CodeGenIP is the insertion point at which the body code should be /// placed. - /// \param ContinuationBB is the basic block target to leave the body. - /// - /// Note that all blocks pointed to by the arguments have terminators. using BodyGenCallbackTy = - function_ref<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, - BasicBlock &ContinuationBB)>; + function_ref<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>; // This is created primarily for sections construct as llvm::function_ref // (BodyGenCallbackTy) is not storable (as described in the comments of // function_ref class - function_ref contains non-ownable reference // to the callable. using StorableBodyGenCallbackTy = - std::function<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, - BasicBlock &ContinuationBB)>; + std::function<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>; /// Callback type for loop body code generation. /// @@ -145,8 +200,7 @@ public: /// Description of a LLVM-IR insertion point (IP) and a debug/source location /// (filename, line, column, ...). struct LocationDescription { - template <typename T, typename U> - LocationDescription(const IRBuilder<T, U> &IRB) + LocationDescription(const IRBuilderBase &IRB) : IP(IRB.saveIP()), DL(IRB.getCurrentDebugLocation()) {} LocationDescription(const InsertPointTy &IP) : IP(IP) {} LocationDescription(const InsertPointTy &IP, const DebugLoc &DL) @@ -345,6 +399,7 @@ public: ArrayRef<CanonicalLoopInfo *> Loops, InsertPointTy ComputeIP); +private: /// Modifies the canonical loop to be a statically-scheduled workshare loop. /// /// This takes a \p LoopInfo representing a canonical loop, such as the one @@ -354,14 +409,6 @@ public: /// the current thread, updates the relevant instructions in the canonical /// loop and calls to an OpenMP runtime finalization function after the loop. /// - /// TODO: Workshare loops with static scheduling may contain up to two loops - /// that fulfill the requirements of an OpenMP canonical loop. One for - /// iterating over all iterations of a chunk and another one for iterating - /// over all chunks that are executed on the same thread. Returning - /// CanonicalLoopInfo objects representing them may eventually be useful for - /// the apply clause planned in OpenMP 6.0, but currently whether these are - /// canonical loops is irrelevant. - /// /// \param DL Debug location for instructions added for the /// workshare-loop construct itself. /// \param CLI A descriptor of the canonical loop to workshare. @@ -369,14 +416,30 @@ public: /// preheader of the loop. /// \param NeedsBarrier Indicates whether a barrier must be inserted after /// the loop. - /// \param Chunk The size of loop chunk considered as a unit when - /// scheduling. If \p nullptr, defaults to 1. /// /// \returns Point where to insert code after the workshare construct. InsertPointTy applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, - bool NeedsBarrier, - Value *Chunk = nullptr); + bool NeedsBarrier); + + /// Modifies the canonical loop a statically-scheduled workshare loop with a + /// user-specified chunk size. + /// + /// \param DL Debug location for instructions added for the + /// workshare-loop construct itself. + /// \param CLI A descriptor of the canonical loop to workshare. + /// \param AllocaIP An insertion point for Alloca instructions usable in + /// the preheader of the loop. + /// \param NeedsBarrier Indicates whether a barrier must be inserted after the + /// loop. + /// \param ChunkSize The user-specified chunk size. + /// + /// \returns Point where to insert code after the workshare construct. + InsertPointTy applyStaticChunkedWorkshareLoop(DebugLoc DL, + CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + bool NeedsBarrier, + Value *ChunkSize); /// Modifies the canonical loop to be a dynamically-scheduled workshare loop. /// @@ -404,6 +467,7 @@ public: bool NeedsBarrier, Value *Chunk = nullptr); +public: /// Modifies the canonical loop to be a workshare loop. /// /// This takes a \p LoopInfo representing a canonical loop, such as the one @@ -413,6 +477,10 @@ public: /// the current thread, updates the relevant instructions in the canonical /// loop and calls to an OpenMP runtime finalization function after the loop. /// + /// The concrete transformation is done by applyStaticWorkshareLoop, + /// applyStaticChunkedWorkshareLoop, or applyDynamicWorkshareLoop, depending + /// on the value of \p SchedKind and \p ChunkSize. + /// /// \param DL Debug location for instructions added for the /// workshare-loop construct itself. /// \param CLI A descriptor of the canonical loop to workshare. @@ -420,10 +488,25 @@ public: /// preheader of the loop. /// \param NeedsBarrier Indicates whether a barrier must be insterted after /// the loop. + /// \param SchedKind Scheduling algorithm to use. + /// \param ChunkSize The chunk size for the inner loop. + /// \param HasSimdModifier Whether the simd modifier is present in the + /// schedule clause. + /// \param HasMonotonicModifier Whether the monotonic modifier is present in + /// the schedule clause. + /// \param HasNonmonotonicModifier Whether the nonmonotonic modifier is + /// present in the schedule clause. + /// \param HasOrderedClause Whether the (parameterless) ordered clause is + /// present. /// /// \returns Point where to insert code after the workshare construct. - InsertPointTy applyWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, - InsertPointTy AllocaIP, bool NeedsBarrier); + InsertPointTy applyWorkshareLoop( + DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, + bool NeedsBarrier, + llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default, + Value *ChunkSize = nullptr, bool HasSimdModifier = false, + bool HasMonotonicModifier = false, bool HasNonmonotonicModifier = false, + bool HasOrderedClause = false); /// Tile a loop nest. /// @@ -535,6 +618,18 @@ public: /// \param Loc The location where the taskyield directive was encountered. void createTaskyield(const LocationDescription &Loc); + /// Generator for `#omp task` + /// + /// \param Loc The location where the task construct was encountered. + /// \param AllocaIP The insertion point to be used for alloca instructions. + /// \param BodyGenCB Callback that will generate the region code. + /// \param Tied True if the task is tied, false if the task is untied. + /// \param Final i1 value which is `true` if the task is final, `false` if the + /// task is not final. + InsertPointTy createTask(const LocationDescription &Loc, + InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, + bool Tied = true, Value *Final = nullptr); + /// Functions used to generate reductions. Such functions take two Values /// representing LHS and RHS of the reduction, respectively, and a reference /// to the value that is updated to refer to the reduction result. @@ -696,6 +791,27 @@ public: /// Value. GlobalValue *createGlobalFlag(unsigned Value, StringRef Name); + /// Create an offloading section struct used to register this global at + /// runtime. + /// + /// Type struct __tgt_offload_entry{ + /// void *addr; // Pointer to the offload entry info. + /// // (function or global) + /// char *name; // Name of the function or global. + /// size_t size; // Size of the entry info (0 if it a function). + /// int32_t flags; + /// int32_t reserved; + /// }; + /// + /// \param Addr The pointer to the global being registered. + /// \param Name The symbol name associated with the global. + /// \param Size The size in bytes of the global (0 for functions). + /// \param Flags Flags associated with the entry. + /// \param SectionName The section this entry will be placed at. + void emitOffloadingEntry(Constant *Addr, StringRef Name, uint64_t Size, + int32_t Flags, + StringRef SectionName = "omp_offloading_entries"); + /// Generate control flow and cleanup for cancellation. /// /// \param CancelFlag Flag indicating if the cancellation is performed. @@ -768,7 +884,7 @@ public: struct OutlineInfo { using PostOutlineCBTy = std::function<void(Function &)>; PostOutlineCBTy PostOutlineCB; - BasicBlock *EntryBB, *ExitBB; + BasicBlock *EntryBB, *ExitBB, *OuterAllocaBB; SmallVector<Value *, 2> ExcludeArgsFromAggregate; /// Collect all blocks in between EntryBB and ExitBB in both the given @@ -851,12 +967,14 @@ public: /// \param Loc The source location description. /// \param BodyGenCB Callback that will generate the region code. /// \param FiniCB Callback to finalize variable copies. + /// \param IsNowait If false, a barrier is emitted. /// \param DidIt Local variable used as a flag to indicate 'single' thread /// /// \returns The insertion position *after* the single call. InsertPointTy createSingle(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB, llvm::Value *DidIt); + FinalizeCallbackTy FiniCB, bool IsNowait, + llvm::Value *DidIt); /// Generator for '#omp master' /// @@ -1198,7 +1316,7 @@ private: const function_ref<Value *(Value *XOld, IRBuilder<> &IRB)>; private: - enum AtomicKind { Read, Write, Update, Capture }; + enum AtomicKind { Read, Write, Update, Capture, Compare }; /// Determine whether to emit flush or not /// @@ -1214,7 +1332,8 @@ private: /// For complex Operations: X = UpdateOp(X) => CmpExch X, old_X, UpdateOp(X) /// Only Scalar data types. /// - /// \param AllocIP Instruction to create AllocaInst before. + /// \param AllocaIP The insertion point to be used for alloca + /// instructions. /// \param X The target atomic pointer to be updated /// \param XElemTy The element type of the atomic pointer. /// \param Expr The value to update X with. @@ -1234,7 +1353,7 @@ private: /// \returns A pair of the old value of X before the update, and the value /// used for the update. std::pair<Value *, Value *> - emitAtomicUpdate(Instruction *AllocIP, Value *X, Type *XElemTy, Value *Expr, + emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr); @@ -1286,7 +1405,7 @@ public: /// Only Scalar data types. /// /// \param Loc The insert and source location description. - /// \param AllocIP Instruction to create AllocaInst before. + /// \param AllocaIP The insertion point to be used for alloca instructions. /// \param X The target atomic pointer to be updated /// \param Expr The value to update X with. /// \param AO Atomic ordering of the generated atomic instructions. @@ -1302,7 +1421,7 @@ public: /// /// \return Insertion point after generated atomic update IR. InsertPointTy createAtomicUpdate(const LocationDescription &Loc, - Instruction *AllocIP, AtomicOpValue &X, + InsertPointTy AllocaIP, AtomicOpValue &X, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, @@ -1317,7 +1436,7 @@ public: /// X = UpdateOp(X); V = X, /// /// \param Loc The insert and source location description. - /// \param AllocIP Instruction to create AllocaInst before. + /// \param AllocaIP The insertion point to be used for alloca instructions. /// \param X The target atomic pointer to be updated /// \param V Memory address where to store captured value /// \param Expr The value to update X with. @@ -1338,12 +1457,63 @@ public: /// /// \return Insertion point after generated atomic capture IR. InsertPointTy - createAtomicCapture(const LocationDescription &Loc, Instruction *AllocIP, + createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, AtomicOpValue &V, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr); + /// Emit atomic compare for constructs: --- Only scalar data types + /// cond-expr-stmt: + /// x = x ordop expr ? expr : x; + /// x = expr ordop x ? expr : x; + /// x = x == e ? d : x; + /// x = e == x ? d : x; (this one is not in the spec) + /// cond-update-stmt: + /// if (x ordop expr) { x = expr; } + /// if (expr ordop x) { x = expr; } + /// if (x == e) { x = d; } + /// if (e == x) { x = d; } (this one is not in the spec) + /// conditional-update-capture-atomic: + /// v = x; cond-update-stmt; (IsPostfixUpdate=true, IsFailOnly=false) + /// cond-update-stmt; v = x; (IsPostfixUpdate=false, IsFailOnly=false) + /// if (x == e) { x = d; } else { v = x; } (IsPostfixUpdate=false, + /// IsFailOnly=true) + /// r = x == e; if (r) { x = d; } (IsPostfixUpdate=false, IsFailOnly=false) + /// r = x == e; if (r) { x = d; } else { v = x; } (IsPostfixUpdate=false, + /// IsFailOnly=true) + /// + /// \param Loc The insert and source location description. + /// \param X The target atomic pointer to be updated. + /// \param V Memory address where to store captured value (for + /// compare capture only). + /// \param R Memory address where to store comparison result + /// (for compare capture with '==' only). + /// \param E The expected value ('e') for forms that use an + /// equality comparison or an expression ('expr') for + /// forms that use 'ordop' (logically an atomic maximum or + /// minimum). + /// \param D The desired value for forms that use an equality + /// comparison. If forms that use 'ordop', it should be + /// \p nullptr. + /// \param AO Atomic ordering of the generated atomic instructions. + /// \param Op Atomic compare operation. It can only be ==, <, or >. + /// \param IsXBinopExpr True if the conditional statement is in the form where + /// x is on LHS. It only matters for < or >. + /// \param IsPostfixUpdate True if original value of 'x' must be stored in + /// 'v', not an updated one (for compare capture + /// only). + /// \param IsFailOnly True if the original value of 'x' is stored to 'v' + /// only when the comparison fails. This is only valid for + /// the case the comparison is '=='. + /// + /// \return Insertion point after generated atomic capture IR. + InsertPointTy + createAtomicCompare(const LocationDescription &Loc, AtomicOpValue &X, + AtomicOpValue &V, AtomicOpValue &R, Value *E, Value *D, + AtomicOrdering AO, omp::OMPAtomicCompareOp Op, + bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly); + /// Create the control flow structure of a canonical OpenMP loop. /// /// The emitted loop will be disconnected, i.e. no edge to the loop's @@ -1484,6 +1654,27 @@ private: /// Re-evaluated whether this makes sense. void collectControlBlocks(SmallVectorImpl<BasicBlock *> &BBs); + /// Sets the number of loop iterations to the given value. This value must be + /// valid in the condition block (i.e., defined in the preheader) and is + /// interpreted as an unsigned integer. + void setTripCount(Value *TripCount); + + /// Replace all uses of the canonical induction variable in the loop body with + /// a new one. + /// + /// The intended use case is to update the induction variable for an updated + /// iteration space such that it can stay normalized in the 0...tripcount-1 + /// range. + /// + /// The \p Updater is called with the (presumable updated) current normalized + /// induction variable and is expected to return the value that uses of the + /// pre-updated induction values should use instead, typically dependent on + /// the new induction variable. This is a lambda (instead of e.g. just passing + /// the new value) to be able to distinguish the uses of the pre-updated + /// induction variable and uses of the induction varible to compute the + /// updated induction variable value. + void mapIndVar(llvm::function_ref<Value *(Instruction *)> Updater); + public: /// Returns whether this object currently represents the IR of a loop. If /// returning false, it may have been consumed by a loop transformation or not diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def index 0c3cb3f43105..14aa53a6b08d 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -86,6 +86,8 @@ __OMP_ARRAY_TYPE(KmpCriticalName, Int32, 8) OMP_STRUCT_TYPE(VarName, "struct." #Name, __VA_ARGS__) __OMP_STRUCT_TYPE(Ident, ident_t, Int32, Int32, Int32, Int32, Int8Ptr) +__OMP_STRUCT_TYPE(OffloadEntry, __tgt_offload_entry, Int8Ptr, Int8Ptr, SizeTy, + Int32, Int32) __OMP_STRUCT_TYPE(AsyncInfo, __tgt_async_info, Int8Ptr) #undef __OMP_STRUCT_TYPE @@ -475,6 +477,7 @@ __OMP_RTL(__last, false, Void, ) #define ParamAttrs(...) ArrayRef<AttributeSet>({__VA_ARGS__}) #define EnumAttr(Kind) Attribute::get(Ctx, Attribute::AttrKind::Kind) #define EnumAttrInt(Kind, N) Attribute::get(Ctx, Attribute::AttrKind::Kind, N) +#define AllocSizeAttr(N, M) Attribute::getWithAllocSizeArgs(Ctx, N, M) #define AttributeSet(...) \ AttributeSet::get(Ctx, ArrayRef<Attribute>({__VA_ARGS__})) @@ -908,8 +911,10 @@ __OMP_RTL_ATTRS(__kmpc_doacross_wait, BarrierAttrs, AttributeSet(), __OMP_RTL_ATTRS(__kmpc_doacross_fini, BarrierAttrs, AttributeSet(), ParamAttrs(ReadOnlyPtrAttrs)) -__OMP_RTL_ATTRS(__kmpc_alloc_shared, DeviceAllocAttrs, ReturnPtrAttrs, - ParamAttrs()) +__OMP_RTL_ATTRS(__kmpc_alloc_shared, AttributeSet( + EnumAttr(NoUnwind), + EnumAttr(NoSync), + AllocSizeAttr(0, None)), ReturnPtrAttrs, ParamAttrs()) __OMP_RTL_ATTRS(__kmpc_free_shared, DeviceAllocAttrs, AttributeSet(), ParamAttrs(NoCaptureAttrs)) @@ -962,6 +967,7 @@ __OMP_RTL_ATTRS(__kmpc_parallel_51, AlwaysInlineAttrs, AttributeSet(), #undef EnumAttr #undef EnumAttrInt #undef ParamAttrs +#undef AllocSizeAttr ///} @@ -1026,6 +1032,7 @@ __OMP_CANCEL_KIND(taskgroup, 4) __OMP_DEFAULT_KIND(none) __OMP_DEFAULT_KIND(shared) +__OMP_DEFAULT_KIND(private) __OMP_DEFAULT_KIND(firstprivate) __OMP_DEFAULT_KIND(unknown) @@ -1153,6 +1160,7 @@ __OMP_TRAIT_PROPERTY(implementation, extension, match_any) __OMP_TRAIT_PROPERTY(implementation, extension, match_none) __OMP_TRAIT_PROPERTY(implementation, extension, disable_implicit_base) __OMP_TRAIT_PROPERTY(implementation, extension, allow_templates) +__OMP_TRAIT_PROPERTY(implementation, extension, bind_to_declaration) __OMP_TRAIT_SET(user) diff --git a/llvm/include/llvm/FuzzMutate/FuzzerCLI.h b/llvm/include/llvm/FuzzMutate/FuzzerCLI.h index 473277396a90..db0168d3e675 100644 --- a/llvm/include/llvm/FuzzMutate/FuzzerCLI.h +++ b/llvm/include/llvm/FuzzMutate/FuzzerCLI.h @@ -14,8 +14,8 @@ #ifndef LLVM_FUZZMUTATE_FUZZERCLI_H #define LLVM_FUZZMUTATE_FUZZERCLI_H -#include "llvm/IR/LLVMContext.h" #include "llvm/Support/DataTypes.h" +#include <stddef.h> namespace llvm { @@ -51,29 +51,6 @@ using FuzzerInitFun = int (*)(int *argc, char ***argv); int runFuzzerOnInputs(int ArgC, char *ArgV[], FuzzerTestFun TestOne, FuzzerInitFun Init = [](int *, char ***) { return 0; }); -/// Fuzzer friendly interface for the llvm bitcode parser. -/// -/// \param Data Bitcode we are going to parse -/// \param Size Size of the 'Data' in bytes -/// \return New module or nullptr in case of error -std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size, - LLVMContext &Context); - -/// Fuzzer friendly interface for the llvm bitcode printer. -/// -/// \param M Module to print -/// \param Dest Location to store serialized module -/// \param MaxSize Size of the destination buffer -/// \return Number of bytes that were written. When module size exceeds MaxSize -/// returns 0 and leaves Dest unchanged. -size_t writeModule(const Module &M, uint8_t *Dest, size_t MaxSize); - -/// Try to parse module and verify it. May output verification errors to the -/// errs(). -/// \return New module or nullptr in case of error. -std::unique_ptr<Module> parseAndVerify(const uint8_t *Data, size_t Size, - LLVMContext &Context); - -} // end llvm namespace +} // namespace llvm #endif // LLVM_FUZZMUTATE_FUZZERCLI_H diff --git a/llvm/include/llvm/FuzzMutate/IRMutator.h b/llvm/include/llvm/FuzzMutate/IRMutator.h index 423582eace9b..ade76f1b5845 100644 --- a/llvm/include/llvm/FuzzMutate/IRMutator.h +++ b/llvm/include/llvm/FuzzMutate/IRMutator.h @@ -10,6 +10,9 @@ // configurable set of strategies. Some common strategies are also included // here. // +// Fuzzer-friendly (de)serialization functions are also provided, as these +// are usually needed when mutating IR. +// //===----------------------------------------------------------------------===// #ifndef LLVM_FUZZMUTATE_IRMUTATOR_H @@ -113,6 +116,29 @@ public: void mutate(Instruction &Inst, RandomIRBuilder &IB) override; }; +/// Fuzzer friendly interface for the llvm bitcode parser. +/// +/// \param Data Bitcode we are going to parse +/// \param Size Size of the 'Data' in bytes +/// \return New module or nullptr in case of error +std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size, + LLVMContext &Context); + +/// Fuzzer friendly interface for the llvm bitcode printer. +/// +/// \param M Module to print +/// \param Dest Location to store serialized module +/// \param MaxSize Size of the destination buffer +/// \return Number of bytes that were written. When module size exceeds MaxSize +/// returns 0 and leaves Dest unchanged. +size_t writeModule(const Module &M, uint8_t *Dest, size_t MaxSize); + +/// Try to parse module and verify it. May output verification errors to the +/// errs(). +/// \return New module or nullptr in case of error. +std::unique_ptr<Module> parseAndVerify(const uint8_t *Data, size_t Size, + LLVMContext &Context); + } // end llvm namespace #endif // LLVM_FUZZMUTATE_IRMUTATOR_H diff --git a/llvm/include/llvm/FuzzMutate/OpDescriptor.h b/llvm/include/llvm/FuzzMutate/OpDescriptor.h index 43c810920766..847f975571bc 100644 --- a/llvm/include/llvm/FuzzMutate/OpDescriptor.h +++ b/llvm/include/llvm/FuzzMutate/OpDescriptor.h @@ -15,16 +15,15 @@ #define LLVM_FUZZMUTATE_OPDESCRIPTOR_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include <functional> namespace llvm { +class Instruction; namespace fuzzerop { /// @{ @@ -146,7 +145,8 @@ static inline SourcePred sizedPtrType() { return false; if (const auto *PtrT = dyn_cast<PointerType>(V->getType())) - return PtrT->getPointerElementType()->isSized(); + return PtrT->isOpaque() || + PtrT->getNonOpaquePointerElementType()->isSized(); return false; }; auto Make = [](ArrayRef<Value *>, ArrayRef<Type *> Ts) { diff --git a/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h b/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h index f3b609702e9d..aeb41baa8e07 100644 --- a/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h +++ b/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h @@ -13,12 +13,19 @@ #ifndef LLVM_FUZZMUTATE_RANDOMIRBUILDER_H #define LLVM_FUZZMUTATE_RANDOMIRBUILDER_H -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/FuzzMutate/IRMutator.h" -#include "llvm/FuzzMutate/Random.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include <random> namespace llvm { +class BasicBlock; +class Instruction; +class LLVMContext; +class Type; +class Value; +namespace fuzzerop { +class SourcePred; +} using RandomEngine = std::mt19937; diff --git a/llvm/include/llvm/IR/AbstractCallSite.h b/llvm/include/llvm/IR/AbstractCallSite.h index 69048554a05c..50afe016f0d6 100644 --- a/llvm/include/llvm/IR/AbstractCallSite.h +++ b/llvm/include/llvm/IR/AbstractCallSite.h @@ -14,17 +14,17 @@ #ifndef LLVM_IR_ABSTRACTCALLSITE_H #define LLVM_IR_ABSTRACTCALLSITE_H -#include "llvm/IR/Argument.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Use.h" #include "llvm/IR/Value.h" -#include "llvm/Support/Casting.h" #include <cassert> namespace llvm { +class Argument; +class Use; + /// AbstractCallSite /// /// An abstract call site is a wrapper that allows to treat direct, diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h index 7cbfa2a7b6ce..3b74853cdafa 100644 --- a/llvm/include/llvm/IR/Argument.h +++ b/llvm/include/llvm/IR/Argument.h @@ -14,7 +14,6 @@ #define LLVM_IR_ARGUMENT_H #include "llvm/ADT/Twine.h" -#include "llvm/ADT/ilist_node.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Value.h" diff --git a/llvm/include/llvm/IR/Assumptions.h b/llvm/include/llvm/IR/Assumptions.h index 08e6c8b6f1e0..2d2ecfbde6e6 100644 --- a/llvm/include/llvm/IR/Assumptions.h +++ b/llvm/include/llvm/IR/Assumptions.h @@ -34,6 +34,10 @@ extern StringSet<> KnownAssumptionStrings; /// Helper that allows to insert a new assumption string in the known assumption /// set by creating a (static) object. struct KnownAssumptionString { + KnownAssumptionString(const char *AssumptionStr) + : AssumptionStr(AssumptionStr) { + KnownAssumptionStrings.insert(AssumptionStr); + } KnownAssumptionString(StringRef AssumptionStr) : AssumptionStr(AssumptionStr) { KnownAssumptionStrings.insert(AssumptionStr); diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h index 74b60f1e3d05..6a4e6d63a973 100644 --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -17,11 +17,13 @@ #include "llvm-c/Types.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/Alignment.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include <bitset> #include <cassert> @@ -42,6 +44,18 @@ class Function; class LLVMContext; class Type; +enum class AllocFnKind : uint64_t { + Unknown = 0, + Alloc = 1 << 0, // Allocator function returns a new allocation + Realloc = 1 << 1, // Allocator function resizes the `allocptr` argument + Free = 1 << 2, // Allocator function frees the `allocptr` argument + Uninitialized = 1 << 3, // Allocator function returns uninitialized memory + Zeroed = 1 << 4, // Allocator function returns zeroed memory + Aligned = 1 << 5, // Allocator function aligns allocations per the + // `allocalign` argument + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Aligned) +}; + //===----------------------------------------------------------------------===// /// \class /// Functions, function parameters, and return types can have attributes @@ -130,6 +144,7 @@ public: static Attribute getWithByRefType(LLVMContext &Context, Type *Ty); static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty); static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty); + static Attribute getWithUWTableKind(LLVMContext &Context, UWTableKind Kind); /// For a typed attribute, return the equivalent attribute with the type /// changed to \p ReplacementTy. @@ -223,6 +238,12 @@ public: /// unknown. Optional<unsigned> getVScaleRangeMax() const; + // Returns the unwind table kind. + UWTableKind getUWTableKind() const; + + // Returns the allocator function kind. + AllocFnKind getAllocKind() const; + /// The Attribute is converted to a string of equivalent mnemonic. This /// is, presumably, for writing out the mnemonics for the assembly writer. std::string getAsString(bool InAttrGrp = false) const; @@ -353,6 +374,8 @@ public: std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; unsigned getVScaleRangeMin() const; Optional<unsigned> getVScaleRangeMax() const; + UWTableKind getUWTableKind() const; + AllocFnKind getAllocKind() const; std::string getAsString(bool InAttrGrp = false) const; /// Return true if this attribute set belongs to the LLVMContext. @@ -841,6 +864,11 @@ public: /// arg. uint64_t getParamDereferenceableOrNullBytes(unsigned ArgNo) const; + /// Get the unwind table kind requested for the function. + UWTableKind getUWTableKind() const; + + AllocFnKind getAllocKind() const; + /// Return the attributes at the index as a string. std::string getAsString(unsigned Index, bool InAttrGrp = false) const; @@ -1190,6 +1218,13 @@ public: /// Attribute.getIntValue(). AttrBuilder &addVScaleRangeAttrFromRawRepr(uint64_t RawVScaleRangeRepr); + /// This turns the unwind table kind into the form used internally in + /// Attribute. + AttrBuilder &addUWTableAttr(UWTableKind Kind); + + // This turns the allocator kind into the form used internally in Attribute. + AttrBuilder &addAllocKindAttr(AllocFnKind Kind); + ArrayRef<Attribute> attrs() const { return Attrs; } bool operator==(const AttrBuilder &B) const; @@ -1198,8 +1233,17 @@ public: namespace AttributeFuncs { -/// Which attributes cannot be applied to a type. -AttributeMask typeIncompatible(Type *Ty); +enum AttributeSafetyKind : uint8_t { + ASK_SAFE_TO_DROP = 1, + ASK_UNSAFE_TO_DROP = 2, + ASK_ALL = ASK_SAFE_TO_DROP | ASK_UNSAFE_TO_DROP, +}; + +/// Which attributes cannot be applied to a type. The argument \p ASK indicates, +/// if only attributes that are known to be safely droppable are contained in +/// the mask; only attributes that might be unsafe to drop (e.g., ABI-related +/// attributes) are in the mask; or both. +AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK = ASK_ALL); /// Get param/return attributes which imply immediate undefined behavior if an /// invalid value is passed. For example, this includes noundef (where undef @@ -1230,6 +1274,9 @@ void mergeAttributesForInlining(Function &Caller, const Function &Callee); /// \param [in] ToMerge - The function to merge attributes from. void mergeAttributesForOutlining(Function &Base, const Function &ToMerge); +/// Update min-legal-vector-width if it is in Attribute and less than Width. +void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width); + } // end namespace AttributeFuncs } // end namespace llvm diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 40c554c269ca..7b955b40b0a8 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -47,6 +47,16 @@ class StrBoolAttr<string S> : Attr<S, []>; /// 0 means unaligned (different from align(1)). def Alignment : IntAttr<"align", [ParamAttr, RetAttr]>; +/// Parameter of a function that tells us the alignment of an allocation, as in +/// aligned_alloc and aligned ::operator::new. +def AllocAlign: EnumAttr<"allocalign", [ParamAttr]>; + +/// Describes behavior of an allocator function in terms of known properties. +def AllocKind: IntAttr<"allockind", [FnAttr]>; + +/// Parameter is the pointer to be manipulated by the allocator function. +def AllocatedPointer : EnumAttr<"allocptr", [ParamAttr]>; + /// The result of the function is guaranteed to point to a number of bytes that /// we can determine if we know the value of the function's arguments. def AllocSize : IntAttr<"allocsize", [FnAttr]>; @@ -175,6 +185,9 @@ def NoProfile : EnumAttr<"noprofile", [FnAttr]>; /// Function doesn't unwind stack. def NoUnwind : EnumAttr<"nounwind", [FnAttr]>; +/// No SanitizeBounds instrumentation. +def NoSanitizeBounds : EnumAttr<"nosanitize_bounds", [FnAttr]>; + /// No SanitizeCoverage instrumentation. def NoSanitizeCoverage : EnumAttr<"nosanitize_coverage", [FnAttr]>; @@ -273,7 +286,7 @@ def SwiftSelf : EnumAttr<"swiftself", [ParamAttr]>; def SwiftAsync : EnumAttr<"swiftasync", [ParamAttr]>; /// Function must be in a unwind table. -def UWTable : EnumAttr<"uwtable", [FnAttr]>; +def UWTable : IntAttr<"uwtable", [FnAttr]>; /// Minimum/Maximum vscale value for function. def VScaleRange : IntAttr<"vscale_range", [FnAttr]>; @@ -290,10 +303,14 @@ def ZExt : EnumAttr<"zeroext", [ParamAttr, RetAttr]>; /// Function is required to make Forward Progress. def MustProgress : EnumAttr<"mustprogress", [FnAttr]>; +/// Function is a presplit coroutine. +def PresplitCoroutine : EnumAttr<"presplitcoroutine", [FnAttr]>; + /// Target-independent string attributes. def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">; def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; +def ApproxFuncFPMath : StrBoolAttr<"approx-func-fp-math">; def NoSignedZerosFPMath : StrBoolAttr<"no-signed-zeros-fp-math">; def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; def NoJumpTables : StrBoolAttr<"no-jump-tables">; @@ -333,6 +350,7 @@ class MergeRule<string F> { def : MergeRule<"setAND<LessPreciseFPMADAttr>">; def : MergeRule<"setAND<NoInfsFPMathAttr>">; def : MergeRule<"setAND<NoNansFPMathAttr>">; +def : MergeRule<"setAND<ApproxFuncFPMathAttr>">; def : MergeRule<"setAND<NoSignedZerosFPMathAttr>">; def : MergeRule<"setAND<UnsafeFPMathAttr>">; def : MergeRule<"setOR<NoImplicitFloatAttr>">; @@ -345,6 +363,3 @@ def : MergeRule<"adjustCallerStackProbeSize">; def : MergeRule<"adjustMinLegalVectorWidth">; def : MergeRule<"adjustNullPointerValidAttr">; def : MergeRule<"setAND<MustProgressAttr>">; - -// Target dependent attributes -include "llvm/IR/AttributesAMDGPU.td" diff --git a/llvm/include/llvm/IR/AttributesAMDGPU.td b/llvm/include/llvm/IR/AttributesAMDGPU.td deleted file mode 100644 index e2a0f045b656..000000000000 --- a/llvm/include/llvm/IR/AttributesAMDGPU.td +++ /dev/null @@ -1,14 +0,0 @@ -//===- AttributesAMDGPU.td - Defines AMDGPU attributes -----*- tablegen -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines AMDGPU specific attributes. -// -//===----------------------------------------------------------------------===// - -def AMDGPUUnsafeFPAtomics : StrBoolAttr<"amdgpu-unsafe-fp-atomics">; -def : MergeRule<"setAND<AMDGPUUnsafeFPAtomicsAttr>">; diff --git a/llvm/include/llvm/IR/AutoUpgrade.h b/llvm/include/llvm/IR/AutoUpgrade.h index f331fc3c413f..12952f25cbda 100644 --- a/llvm/include/llvm/IR/AutoUpgrade.h +++ b/llvm/include/llvm/IR/AutoUpgrade.h @@ -14,19 +14,24 @@ #define LLVM_IR_AUTOUPGRADE_H #include "llvm/ADT/StringRef.h" +#include <vector> namespace llvm { class AttrBuilder; - class CallInst; + class CallBase; class Constant; class Function; class Instruction; + class GlobalVariable; class MDNode; class Module; - class GlobalVariable; + class StringRef; class Type; class Value; + template <typename T> class OperandBundleDefT; + using OperandBundleDef = OperandBundleDefT<Value *>; + /// This is a more granular function that simply checks an intrinsic function /// for upgrading, and returns true if it requires upgrading. It may return /// null in NewFn if the all calls to the original intrinsic function @@ -35,7 +40,7 @@ namespace llvm { /// This is the complement to the above, replacing a specific call to an /// intrinsic function with a call to the specified new function. - void UpgradeIntrinsicCall(CallInst *CI, Function *NewFn); + void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn); // This upgrades the comment for objc retain release markers in inline asm // calls @@ -77,7 +82,7 @@ namespace llvm { /// This is an auto-upgrade for bitcast constant expression between pointers /// with different address spaces: the instruction is replaced by a pair /// ptrtoint+inttoptr. - Value *UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy); + Constant *UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy); /// Check the debug info version number, if it is out-dated, drop the debug /// info. Return true if module is modified. @@ -98,6 +103,9 @@ namespace llvm { /// Upgrade attributes that changed format or kind. void UpgradeAttributes(AttrBuilder &B); + /// Upgrade operand bundles (without knowing about their user instruction). + void UpgradeOperandBundles(std::vector<OperandBundleDef> &OperandBundles); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h index 184ddfc01c29..d487223eca02 100644 --- a/llvm/include/llvm/IR/BasicBlock.h +++ b/llvm/include/llvm/IR/BasicBlock.h @@ -22,9 +22,6 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/SymbolTableListTraits.h" #include "llvm/IR/Value.h" -#include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Compiler.h" #include <cassert> #include <cstddef> #include <iterator> @@ -119,7 +116,11 @@ public: /// Returns the terminator instruction if the block is well formed or null /// if the block is not well formed. - const Instruction *getTerminator() const LLVM_READONLY; + const Instruction *getTerminator() const LLVM_READONLY { + if (InstList.empty() || !InstList.back().isTerminator()) + return nullptr; + return &InstList.back(); + } Instruction *getTerminator() { return const_cast<Instruction *>( static_cast<const BasicBlock *>(this)->getTerminator()); diff --git a/llvm/include/llvm/IR/CFG.h b/llvm/include/llvm/IR/CFG.h index 0ee584f8af7e..28a8d31a4cc6 100644 --- a/llvm/include/llvm/IR/CFG.h +++ b/llvm/include/llvm/IR/CFG.h @@ -25,7 +25,6 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/Value.h" -#include "llvm/Support/Casting.h" #include <cassert> #include <cstddef> #include <iterator> diff --git a/llvm/include/llvm/IR/ConstantFold.h b/llvm/include/llvm/IR/ConstantFold.h new file mode 100644 index 000000000000..d637a180b0ba --- /dev/null +++ b/llvm/include/llvm/IR/ConstantFold.h @@ -0,0 +1,60 @@ +//==-- ConstantFold.h - DL-independent Constant Folding Interface -*- C++ -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the DataLayout-independent constant folding interface. +// When possible, the DataLayout-aware constant folding interface in +// Analysis/ConstantFolding.h should be preferred. +// +// These interfaces are used by the ConstantExpr::get* methods to automatically +// fold constants when possible. +// +// These operators may return a null object if they don't know how to perform +// the specified operation on the specified constant types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_CONSTANTFOLD_H +#define LLVM_IR_CONSTANTFOLD_H + +#include "llvm/ADT/Optional.h" +#include "llvm/IR/InstrTypes.h" + +namespace llvm { + template <typename T> class ArrayRef; + class Value; + class Constant; + class Type; + + // Constant fold various types of instruction... + Constant *ConstantFoldCastInstruction( + unsigned opcode, ///< The opcode of the cast + Constant *V, ///< The source constant + Type *DestTy ///< The destination type + ); + Constant *ConstantFoldSelectInstruction(Constant *Cond, + Constant *V1, Constant *V2); + Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx); + Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt, + Constant *Idx); + Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, + ArrayRef<int> Mask); + Constant *ConstantFoldExtractValueInstruction(Constant *Agg, + ArrayRef<unsigned> Idxs); + Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, + ArrayRef<unsigned> Idxs); + Constant *ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V); + Constant *ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, + Constant *V2); + Constant *ConstantFoldCompareInstruction(CmpInst::Predicate Predicate, + Constant *C1, Constant *C2); + Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool InBounds, + Optional<unsigned> InRangeIndex, + ArrayRef<Value *> Idxs); +} // End llvm namespace + +#endif diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h index 28dc63a5886e..5e7ddb9aa673 100644 --- a/llvm/include/llvm/IR/ConstantFolder.h +++ b/llvm/include/llvm/IR/ConstantFolder.h @@ -19,9 +19,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/ConstantFold.h" #include "llvm/IR/IRBuilderFolder.h" -#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Operator.h" namespace llvm { @@ -38,31 +39,46 @@ public: // Return an existing value or a constant if the operation can be simplified. // Otherwise return nullptr. //===--------------------------------------------------------------------===// - Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, - bool HasNSW = false) const override { + + Value *FoldBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); if (LC && RC) - return ConstantExpr::getAdd(LC, RC, HasNUW, HasNSW); + return ConstantExpr::get(Opc, LC, RC); return nullptr; } - Value *FoldAnd(Value *LHS, Value *RHS) const override { + Value *FoldExactBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool IsExact) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); if (LC && RC) - return ConstantExpr::getAnd(LC, RC); + return ConstantExpr::get(Opc, LC, RC, + IsExact ? PossiblyExactOperator::IsExact : 0); return nullptr; } - Value *FoldOr(Value *LHS, Value *RHS) const override { + Value *FoldNoWrapBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool HasNUW, bool HasNSW) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); - if (LC && RC) - return ConstantExpr::getOr(LC, RC); + if (LC && RC) { + unsigned Flags = 0; + if (HasNUW) + Flags |= OverflowingBinaryOperator::NoUnsignedWrap; + if (HasNSW) + Flags |= OverflowingBinaryOperator::NoSignedWrap; + return ConstantExpr::get(Opc, LC, RC, Flags); + } return nullptr; } + Value *FoldBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + FastMathFlags FMF) const override { + return FoldBinOp(Opc, LHS, RHS); + } + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { auto *LC = dyn_cast<Constant>(LHS); auto *RC = dyn_cast<Constant>(RHS); @@ -95,103 +111,57 @@ public: return nullptr; } - //===--------------------------------------------------------------------===// - // Binary Operators - //===--------------------------------------------------------------------===// - - Constant *CreateFAdd(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getFAdd(LHS, RHS); - } - - Constant *CreateSub(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return ConstantExpr::getSub(LHS, RHS, HasNUW, HasNSW); - } - - Constant *CreateFSub(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getFSub(LHS, RHS); - } - - Constant *CreateMul(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return ConstantExpr::getMul(LHS, RHS, HasNUW, HasNSW); - } - - Constant *CreateFMul(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getFMul(LHS, RHS); - } - - Constant *CreateUDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstantExpr::getUDiv(LHS, RHS, isExact); - } - - Constant *CreateSDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstantExpr::getSDiv(LHS, RHS, isExact); - } - - Constant *CreateFDiv(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getFDiv(LHS, RHS); - } - - Constant *CreateURem(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getURem(LHS, RHS); - } - - Constant *CreateSRem(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getSRem(LHS, RHS); - } - - Constant *CreateFRem(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getFRem(LHS, RHS); - } - - Constant *CreateShl(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return ConstantExpr::getShl(LHS, RHS, HasNUW, HasNSW); - } - - Constant *CreateLShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstantExpr::getLShr(LHS, RHS, isExact); - } - - Constant *CreateAShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - return ConstantExpr::getAShr(LHS, RHS, isExact); + Value *FoldExtractValue(Value *Agg, + ArrayRef<unsigned> IdxList) const override { + if (auto *CAgg = dyn_cast<Constant>(Agg)) + return ConstantFoldExtractValueInstruction(CAgg, IdxList); + return nullptr; + }; + + Value *FoldInsertValue(Value *Agg, Value *Val, + ArrayRef<unsigned> IdxList) const override { + auto *CAgg = dyn_cast<Constant>(Agg); + auto *CVal = dyn_cast<Constant>(Val); + if (CAgg && CVal) + return ConstantFoldInsertValueInstruction(CAgg, CVal, IdxList); + return nullptr; } - Constant *CreateOr(Constant *LHS, Constant *RHS) const { - return ConstantExpr::getOr(LHS, RHS); + Value *FoldExtractElement(Value *Vec, Value *Idx) const override { + auto *CVec = dyn_cast<Constant>(Vec); + auto *CIdx = dyn_cast<Constant>(Idx); + if (CVec && CIdx) + return ConstantExpr::getExtractElement(CVec, CIdx); + return nullptr; } - Constant *CreateXor(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getXor(LHS, RHS); + Value *FoldInsertElement(Value *Vec, Value *NewElt, + Value *Idx) const override { + auto *CVec = dyn_cast<Constant>(Vec); + auto *CNewElt = dyn_cast<Constant>(NewElt); + auto *CIdx = dyn_cast<Constant>(Idx); + if (CVec && CNewElt && CIdx) + return ConstantExpr::getInsertElement(CVec, CNewElt, CIdx); + return nullptr; } - Constant *CreateBinOp(Instruction::BinaryOps Opc, - Constant *LHS, Constant *RHS) const override { - return ConstantExpr::get(Opc, LHS, RHS); + Value *FoldShuffleVector(Value *V1, Value *V2, + ArrayRef<int> Mask) const override { + auto *C1 = dyn_cast<Constant>(V1); + auto *C2 = dyn_cast<Constant>(V2); + if (C1 && C2) + return ConstantExpr::getShuffleVector(C1, C2, Mask); + return nullptr; } //===--------------------------------------------------------------------===// // Unary Operators //===--------------------------------------------------------------------===// - Constant *CreateNeg(Constant *C, - bool HasNUW = false, bool HasNSW = false) const override { - return ConstantExpr::getNeg(C, HasNUW, HasNSW); - } - Constant *CreateFNeg(Constant *C) const override { return ConstantExpr::getFNeg(C); } - Constant *CreateNot(Constant *C) const override { - return ConstantExpr::getNot(C); - } - Constant *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const override { return ConstantExpr::get(Opc, C); } @@ -255,34 +225,6 @@ public: Constant *RHS) const override { return ConstantExpr::getCompare(P, LHS, RHS); } - - //===--------------------------------------------------------------------===// - // Other Instructions - //===--------------------------------------------------------------------===// - - Constant *CreateExtractElement(Constant *Vec, Constant *Idx) const override { - return ConstantExpr::getExtractElement(Vec, Idx); - } - - Constant *CreateInsertElement(Constant *Vec, Constant *NewElt, - Constant *Idx) const override { - return ConstantExpr::getInsertElement(Vec, NewElt, Idx); - } - - Constant *CreateShuffleVector(Constant *V1, Constant *V2, - ArrayRef<int> Mask) const override { - return ConstantExpr::getShuffleVector(V1, V2, Mask); - } - - Constant *CreateExtractValue(Constant *Agg, - ArrayRef<unsigned> IdxList) const override { - return ConstantExpr::getExtractValue(Agg, IdxList); - } - - Constant *CreateInsertValue(Constant *Agg, Constant *Val, - ArrayRef<unsigned> IdxList) const override { - return ConstantExpr::getInsertValue(Agg, Val, IdxList); - } }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h index fea4d0da1d0d..68abf4ef555d 100644 --- a/llvm/include/llvm/IR/ConstantRange.h +++ b/llvm/include/llvm/IR/ConstantRange.h @@ -553,6 +553,9 @@ public: /// Return whether unsigned mul of the two ranges always/never overflows. OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const; + /// Return known bits for values in this range. + KnownBits toKnownBits() const; + /// Print out the bounds to a stream. void print(raw_ostream &OS) const; diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index fb884912b318..b5445ff71b74 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -289,7 +289,8 @@ public: APInt *Payload = nullptr); static Constant *getSNaN(Type *Ty, bool Negative = false, APInt *Payload = nullptr); - static Constant *getNegativeZero(Type *Ty); + static Constant *getZero(Type *Ty, bool Negative = false); + static Constant *getNegativeZero(Type *Ty) { return getZero(Ty, true); } static Constant *getInfinity(Type *Ty, bool Negative = false); /// Return true if Ty is big enough to represent V. @@ -1120,9 +1121,12 @@ public: /// commutative, callers can acquire the operand 1 identity constant by /// setting AllowRHSConstant to true. For example, any shift has a zero /// identity constant for operand 1: X shift 0 = X. + /// If this is a fadd/fsub operation and we don't care about signed zeros, + /// then setting NSZ to true returns the identity +0.0 instead of -0.0. /// Return nullptr if the operator does not have an identity constant. static Constant *getBinOpIdentity(unsigned Opcode, Type *Ty, - bool AllowRHSConstant = false); + bool AllowRHSConstant = false, + bool NSZ = false); /// Return the absorbing element for the given binary /// operation, i.e. a constant C such that X op C = C and C op X = C for @@ -1160,6 +1164,11 @@ public: Type *Ty ///< The type to trunc or bitcast C to ); + /// Create either an sext, trunc or nothing, depending on whether Ty is + /// wider, narrower or the same as C->getType(). This only works with + /// integer or vector of integer types. + static Constant *getSExtOrTrunc(Constant *C, Type *Ty); + /// Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant /// expression. static Constant * @@ -1285,8 +1294,6 @@ public: static Constant *getShuffleVector(Constant *V1, Constant *V2, ArrayRef<int> Mask, Type *OnlyIfReducedTy = nullptr); - static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs, - Type *OnlyIfReducedTy = nullptr); static Constant *getInsertValue(Constant *Agg, Constant *Val, ArrayRef<unsigned> Idxs, Type *OnlyIfReducedTy = nullptr); diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h index fc461fc3f49f..9afa715b650c 100644 --- a/llvm/include/llvm/IR/DIBuilder.h +++ b/llvm/include/llvm/IR/DIBuilder.h @@ -21,6 +21,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/TrackingMDRef.h" #include "llvm/Support/Casting.h" @@ -220,6 +221,23 @@ namespace llvm { /// \param SizeInBits Size of the type. DIStringType *createStringType(StringRef Name, uint64_t SizeInBits); + /// Create debugging information entry for Fortran + /// assumed length string type. + /// \param Name Type name. + /// \param StringLength String length expressed as DIVariable *. + /// \param StrLocationExp Optional memory location of the string. + DIStringType *createStringType(StringRef Name, DIVariable *StringLength, + DIExpression *StrLocationExp = nullptr); + + /// Create debugging information entry for Fortran + /// assumed length string type. + /// \param Name Type name. + /// \param StringLengthExp String length expressed in DIExpression form. + /// \param StrLocationExp Optional memory location of the string. + DIStringType *createStringType(StringRef Name, + DIExpression *StringLengthExp, + DIExpression *StrLocationExp = nullptr); + /// Create debugging information entry for a qualified /// type, e.g. 'const int'. /// \param Tag Tag identifing type, e.g. dwarf::TAG_volatile_type @@ -734,6 +752,8 @@ namespace llvm { /// \param TParams Function template parameters. /// \param ThrownTypes Exception types this function may throw. /// \param Annotations Attribute Annotations. + /// \param TargetFuncName The name of the target function if this is + /// a trampoline. DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, @@ -742,7 +762,8 @@ namespace llvm { DITemplateParameterArray TParams = nullptr, DISubprogram *Decl = nullptr, DITypeArray ThrownTypes = nullptr, - DINodeArray Annotations = nullptr); + DINodeArray Annotations = nullptr, + StringRef TargetFuncName = ""); /// Identical to createFunction, /// except that the resulting DbgNode is meant to be RAUWed. diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h index 36438fc4f4e0..a6621c963d85 100644 --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -26,10 +26,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Type.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/Alignment.h" #include "llvm/Support/TrailingObjects.h" #include "llvm/Support/TypeSize.h" #include <cassert> diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index 96569179060f..db1d031a062d 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -22,7 +22,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Casting.h" @@ -61,6 +60,10 @@ namespace llvm { +namespace dwarf { +enum Tag : uint16_t; +} + extern cl::opt<bool> EnableFSDiscriminator; class DITypeRefArray { @@ -156,7 +159,7 @@ protected: void setTag(unsigned Tag) { SubclassData16 = Tag; } public: - dwarf::Tag getTag() const { return (dwarf::Tag)SubclassData16; } + dwarf::Tag getTag() const; /// Debug info flags. /// @@ -267,7 +270,7 @@ public: /// Return a (temporary) clone of this. TempGenericDINode clone() const { return cloneImpl(); } - dwarf::Tag getTag() const { return (dwarf::Tag)SubclassData16; } + dwarf::Tag getTag() const; StringRef getHeader() const { return getStringOperand(0); } MDString *getRawHeader() const { return getOperandAs<MDString>(0); } @@ -298,8 +301,7 @@ class DISubrange : public DINode { friend class LLVMContextImpl; friend class MDNode; - DISubrange(LLVMContext &C, StorageType Storage, ArrayRef<Metadata *> Ops) - : DINode(C, DISubrangeKind, Storage, dwarf::DW_TAG_subrange_type, Ops) {} + DISubrange(LLVMContext &C, StorageType Storage, ArrayRef<Metadata *> Ops); ~DISubrange() = default; @@ -363,9 +365,7 @@ class DIGenericSubrange : public DINode { friend class MDNode; DIGenericSubrange(LLVMContext &C, StorageType Storage, - ArrayRef<Metadata *> Ops) - : DINode(C, DIGenericSubrangeKind, Storage, - dwarf::DW_TAG_generic_subrange, Ops) {} + ArrayRef<Metadata *> Ops); ~DIGenericSubrange() = default; @@ -414,11 +414,7 @@ class DIEnumerator : public DINode { APInt Value; DIEnumerator(LLVMContext &C, StorageType Storage, const APInt &Value, - bool IsUnsigned, ArrayRef<Metadata *> Ops) - : DINode(C, DIEnumeratorKind, Storage, dwarf::DW_TAG_enumerator, Ops), - Value(Value) { - SubclassData32 = IsUnsigned; - } + bool IsUnsigned, ArrayRef<Metadata *> Ops); DIEnumerator(LLVMContext &C, StorageType Storage, int64_t Value, bool IsUnsigned, ArrayRef<Metadata *> Ops) : DIEnumerator(C, Storage, APInt(64, Value, !IsUnsigned), IsUnsigned, @@ -568,9 +564,7 @@ private: DIFile(LLVMContext &C, StorageType Storage, Optional<ChecksumInfo<MDString *>> CS, Optional<MDString *> Src, - ArrayRef<Metadata *> Ops) - : DIScope(C, DIFileKind, Storage, dwarf::DW_TAG_file_type, Ops), - Checksum(CS), Source(Src) {} + ArrayRef<Metadata *> Ops); ~DIFile() = default; static DIFile *getImpl(LLVMContext &Context, StringRef Filename, @@ -1021,42 +1015,19 @@ public: /// Get casted version of extra data. /// @{ - DIType *getClassType() const { - assert(getTag() == dwarf::DW_TAG_ptr_to_member_type); - return cast_or_null<DIType>(getExtraData()); - } + DIType *getClassType() const; DIObjCProperty *getObjCProperty() const { return dyn_cast_or_null<DIObjCProperty>(getExtraData()); } - uint32_t getVBPtrOffset() const { - assert(getTag() == dwarf::DW_TAG_inheritance); - if (auto *CM = cast_or_null<ConstantAsMetadata>(getExtraData())) - if (auto *CI = dyn_cast_or_null<ConstantInt>(CM->getValue())) - return static_cast<uint32_t>(CI->getZExtValue()); - return 0; - } + uint32_t getVBPtrOffset() const; - Constant *getStorageOffsetInBits() const { - assert(getTag() == dwarf::DW_TAG_member && isBitField()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) - return C->getValue(); - return nullptr; - } + Constant *getStorageOffsetInBits() const; - Constant *getConstant() const { - assert(getTag() == dwarf::DW_TAG_member && isStaticMember()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) - return C->getValue(); - return nullptr; - } - Constant *getDiscriminantValue() const { - assert(getTag() == dwarf::DW_TAG_member && !isStaticMember()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) - return C->getValue(); - return nullptr; - } + Constant *getConstant() const; + + Constant *getDiscriminantValue() const; /// @} static bool classof(const Metadata *MD) { @@ -1300,10 +1271,7 @@ class DISubroutineType : public DIType { uint8_t CC; DISubroutineType(LLVMContext &C, StorageType Storage, DIFlags Flags, - uint8_t CC, ArrayRef<Metadata *> Ops) - : DIType(C, DISubroutineTypeKind, Storage, dwarf::DW_TAG_subroutine_type, - 0, 0, 0, 0, Flags, Ops), - CC(CC) {} + uint8_t CC, ArrayRef<Metadata *> Ops); ~DISubroutineType() = default; static DISubroutineType *getImpl(LLVMContext &Context, DIFlags Flags, @@ -1330,6 +1298,12 @@ public: (Flags, CC, TypeArray)) TempDISubroutineType clone() const { return cloneImpl(); } + // Returns a new temporary DISubroutineType with updated CC + TempDISubroutineType cloneWithCC(uint8_t CC) const { + auto NewTy = clone(); + NewTy->CC = CC; + return NewTy; + } uint8_t getCC() const { return CC; } @@ -1385,15 +1359,7 @@ private: bool IsOptimized, unsigned RuntimeVersion, unsigned EmissionKind, uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling, unsigned NameTableKind, - bool RangesBaseAddress, ArrayRef<Metadata *> Ops) - : DIScope(C, DICompileUnitKind, Storage, dwarf::DW_TAG_compile_unit, Ops), - SourceLanguage(SourceLanguage), IsOptimized(IsOptimized), - RuntimeVersion(RuntimeVersion), EmissionKind(EmissionKind), - DWOId(DWOId), SplitDebugInlining(SplitDebugInlining), - DebugInfoForProfiling(DebugInfoForProfiling), - NameTableKind(NameTableKind), RangesBaseAddress(RangesBaseAddress) { - assert(Storage != Uniqued); - } + bool RangesBaseAddress, ArrayRef<Metadata *> Ops); ~DICompileUnit() = default; static DICompileUnit * @@ -1872,19 +1838,7 @@ public: static DISPFlags toSPFlags(bool IsLocalToUnit, bool IsDefinition, bool IsOptimized, unsigned Virtuality = SPFlagNonvirtual, - bool IsMainSubprogram = false) { - // We're assuming virtuality is the low-order field. - static_assert(int(SPFlagVirtual) == int(dwarf::DW_VIRTUALITY_virtual) && - int(SPFlagPureVirtual) == - int(dwarf::DW_VIRTUALITY_pure_virtual), - "Virtuality constant mismatch"); - return static_cast<DISPFlags>( - (Virtuality & SPFlagVirtuality) | - (IsLocalToUnit ? SPFlagLocalToUnit : SPFlagZero) | - (IsDefinition ? SPFlagDefinition : SPFlagZero) | - (IsOptimized ? SPFlagOptimized : SPFlagZero) | - (IsMainSubprogram ? SPFlagMainSubprogram : SPFlagZero)); - } + bool IsMainSubprogram = false); private: DIFlags Flags; @@ -1892,13 +1846,7 @@ private: DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line, unsigned ScopeLine, unsigned VirtualIndex, int ThisAdjustment, - DIFlags Flags, DISPFlags SPFlags, ArrayRef<Metadata *> Ops) - : DILocalScope(C, DISubprogramKind, Storage, dwarf::DW_TAG_subprogram, - Ops), - Line(Line), ScopeLine(ScopeLine), VirtualIndex(VirtualIndex), - ThisAdjustment(ThisAdjustment), Flags(Flags), SPFlags(SPFlags) { - static_assert(dwarf::DW_VIRTUALITY_max < 4, "Virtuality out of range"); - } + DIFlags Flags, DISPFlags SPFlags, ArrayRef<Metadata *> Ops); ~DISubprogram() = default; static DISubprogram * @@ -1909,13 +1857,14 @@ private: DISPFlags SPFlags, DICompileUnit *Unit, DITemplateParameterArray TemplateParams, DISubprogram *Declaration, DINodeArray RetainedNodes, DITypeArray ThrownTypes, - DINodeArray Annotations, StorageType Storage, - bool ShouldCreate = true) { + DINodeArray Annotations, StringRef TargetFuncName, + StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Scope, getCanonicalMDString(Context, Name), getCanonicalMDString(Context, LinkageName), File, Line, Type, ScopeLine, ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams.get(), Declaration, RetainedNodes.get(), ThrownTypes.get(), Annotations.get(), + getCanonicalMDString(Context, TargetFuncName), Storage, ShouldCreate); } static DISubprogram * @@ -1925,7 +1874,8 @@ private: int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, Metadata *Unit, Metadata *TemplateParams, Metadata *Declaration, Metadata *RetainedNodes, Metadata *ThrownTypes, Metadata *Annotations, - StorageType Storage, bool ShouldCreate = true); + MDString *TargetFuncName, StorageType Storage, + bool ShouldCreate = true); TempDISubprogram cloneImpl() const { return getTemporary(getContext(), getScope(), getName(), getLinkageName(), @@ -1933,7 +1883,8 @@ private: getContainingType(), getVirtualIndex(), getThisAdjustment(), getFlags(), getSPFlags(), getUnit(), getTemplateParams(), getDeclaration(), - getRetainedNodes(), getThrownTypes(), getAnnotations()); + getRetainedNodes(), getThrownTypes(), getAnnotations(), + getTargetFuncName()); } public: @@ -1945,10 +1896,11 @@ public: DIFlags Flags, DISPFlags SPFlags, DICompileUnit *Unit, DITemplateParameterArray TemplateParams = nullptr, DISubprogram *Declaration = nullptr, DINodeArray RetainedNodes = nullptr, - DITypeArray ThrownTypes = nullptr, DINodeArray Annotations = nullptr), + DITypeArray ThrownTypes = nullptr, DINodeArray Annotations = nullptr, + StringRef TargetFuncName = ""), (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams, - Declaration, RetainedNodes, ThrownTypes, Annotations)) + Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName)) DEFINE_MDNODE_GET( DISubprogram, @@ -1958,10 +1910,10 @@ public: DIFlags Flags, DISPFlags SPFlags, Metadata *Unit, Metadata *TemplateParams = nullptr, Metadata *Declaration = nullptr, Metadata *RetainedNodes = nullptr, Metadata *ThrownTypes = nullptr, - Metadata *Annotations = nullptr), + Metadata *Annotations = nullptr, MDString *TargetFuncName = nullptr), (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams, - Declaration, RetainedNodes, ThrownTypes, Annotations)) + Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName)) TempDISubprogram clone() const { return cloneImpl(); } @@ -2050,6 +2002,10 @@ public: DIType *getContainingType() const { return cast_or_null<DIType>(getRawContainingType()); } + void replaceType(DISubroutineType *Ty) { + assert(isDistinct() && "Only distinct nodes can mutate"); + replaceOperandWith(4, Ty); + } DICompileUnit *getUnit() const { return cast_or_null<DICompileUnit>(getRawUnit()); @@ -2070,6 +2026,9 @@ public: DINodeArray getAnnotations() const { return cast_or_null<MDTuple>(getRawAnnotations()); } + StringRef getTargetFuncName() const { + return (getRawTargetFuncName()) ? getStringOperand(12) : StringRef(); + } Metadata *getRawScope() const { return getOperand(1); } MDString *getRawName() const { return getOperandAs<MDString>(2); } @@ -2090,6 +2049,9 @@ public: Metadata *getRawAnnotations() const { return getNumOperands() > 11 ? getOperandAs<Metadata>(11) : nullptr; } + MDString *getRawTargetFuncName() const { + return getNumOperands() > 12 ? getOperandAs<MDString>(12) : nullptr; + } void replaceRawLinkageName(MDString *LinkageName) { replaceOperandWith(3, LinkageName); @@ -2108,8 +2070,7 @@ public: class DILexicalBlockBase : public DILocalScope { protected: DILexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage, - ArrayRef<Metadata *> Ops) - : DILocalScope(C, ID, Storage, dwarf::DW_TAG_lexical_block, Ops) {} + ArrayRef<Metadata *> Ops); ~DILexicalBlockBase() = default; public: @@ -2301,10 +2262,7 @@ class DINamespace : public DIScope { unsigned ExportSymbols : 1; DINamespace(LLVMContext &Context, StorageType Storage, bool ExportSymbols, - ArrayRef<Metadata *> Ops) - : DIScope(Context, DINamespaceKind, Storage, dwarf::DW_TAG_namespace, - Ops), - ExportSymbols(ExportSymbols) {} + ArrayRef<Metadata *> Ops); ~DINamespace() = default; static DINamespace *getImpl(LLVMContext &Context, DIScope *Scope, @@ -2353,9 +2311,7 @@ class DIModule : public DIScope { bool IsDecl; DIModule(LLVMContext &Context, StorageType Storage, unsigned LineNo, - bool IsDecl, ArrayRef<Metadata *> Ops) - : DIScope(Context, DIModuleKind, Storage, dwarf::DW_TAG_module, Ops), - LineNo(LineNo), IsDecl(IsDecl) {} + bool IsDecl, ArrayRef<Metadata *> Ops); ~DIModule() = default; static DIModule *getImpl(LLVMContext &Context, DIFile *File, DIScope *Scope, @@ -2449,10 +2405,7 @@ class DITemplateTypeParameter : public DITemplateParameter { friend class MDNode; DITemplateTypeParameter(LLVMContext &Context, StorageType Storage, - bool IsDefault, ArrayRef<Metadata *> Ops) - : DITemplateParameter(Context, DITemplateTypeParameterKind, Storage, - dwarf::DW_TAG_template_type_parameter, IsDefault, - Ops) {} + bool IsDefault, ArrayRef<Metadata *> Ops); ~DITemplateTypeParameter() = default; static DITemplateTypeParameter *getImpl(LLVMContext &Context, StringRef Name, @@ -2541,10 +2494,8 @@ class DIVariable : public DINode { uint32_t AlignInBits; protected: - DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Line, - ArrayRef<Metadata *> Ops, uint32_t AlignInBits = 0) - : DINode(C, ID, Storage, dwarf::DW_TAG_variable, Ops), Line(Line), - AlignInBits(AlignInBits) {} + DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, signed Line, + ArrayRef<Metadata *> Ops, uint32_t AlignInBits = 0); ~DIVariable() = default; public: @@ -2763,9 +2714,7 @@ public: } /// Return whether the first element a DW_OP_deref. - bool startsWithDeref() const { - return getNumElements() > 0 && getElement(0) == dwarf::DW_OP_deref; - } + bool startsWithDeref() const; /// Holds the characteristics of one fragment of a larger variable. struct FragmentInfo { @@ -2783,7 +2732,7 @@ public: } /// Return whether this is a piece of an aggregate variable. - bool isFragment() const { return getFragmentInfo().hasValue(); } + bool isFragment() const { return getFragmentInfo().has_value(); } /// Return whether this is an implicit location description. bool isImplicit() const; @@ -2923,10 +2872,7 @@ public: /// Check if the expression consists of exactly one entry value operand. /// (This is the only configuration of entry values that is supported.) - bool isEntryValue() const { - return getNumElements() > 0 && - getElement(0) == dwarf::DW_OP_LLVM_entry_value; - } + bool isEntryValue() const; /// Try to shorten an expression with an initial constant operand. /// Returns a new expression and constant on success, or the original @@ -3057,10 +3003,7 @@ class DICommonBlock : public DIScope { friend class MDNode; DICommonBlock(LLVMContext &Context, StorageType Storage, unsigned LineNo, - ArrayRef<Metadata *> Ops) - : DIScope(Context, DICommonBlockKind, Storage, dwarf::DW_TAG_common_block, - Ops), - LineNo(LineNo) {} + ArrayRef<Metadata *> Ops); static DICommonBlock *getImpl(LLVMContext &Context, DIScope *Scope, DIGlobalVariable *Decl, StringRef Name, @@ -3209,8 +3152,7 @@ class DILabel : public DINode { unsigned Line; DILabel(LLVMContext &C, StorageType Storage, unsigned Line, - ArrayRef<Metadata *> Ops) - : DINode(C, DILabelKind, Storage, dwarf::DW_TAG_label, Ops), Line(Line) {} + ArrayRef<Metadata *> Ops); ~DILabel() = default; static DILabel *getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name, @@ -3276,10 +3218,7 @@ class DIObjCProperty : public DINode { unsigned Attributes; DIObjCProperty(LLVMContext &C, StorageType Storage, unsigned Line, - unsigned Attributes, ArrayRef<Metadata *> Ops) - : DINode(C, DIObjCPropertyKind, Storage, dwarf::DW_TAG_APPLE_property, - Ops), - Line(Line), Attributes(Attributes) {} + unsigned Attributes, ArrayRef<Metadata *> Ops); ~DIObjCProperty() = default; static DIObjCProperty * @@ -3705,7 +3644,7 @@ public: const DILocation *getInlinedAt() const { return InlinedAt; } FragmentInfo getFragmentOrDefault() const { - return Fragment.getValueOr(DefaultFragment); + return Fragment.value_or(DefaultFragment); } static bool isDefaultFragment(const FragmentInfo F) { diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h index f52ce3cde318..f505fd3f3e32 100644 --- a/llvm/include/llvm/IR/DerivedTypes.h +++ b/llvm/include/llvm/IR/DerivedTypes.h @@ -659,7 +659,7 @@ public: } /// This constructs a pointer type with the same pointee type as input - /// PointerType (or opaque pointer is the input PointerType is opaque) and the + /// PointerType (or opaque pointer if the input PointerType is opaque) and the /// given address space. This is only useful during the opaque pointer /// transition. /// TODO: remove after opaque pointer transition is complete. @@ -670,13 +670,6 @@ public: return get(PT->PointeeTy, AddressSpace); } - [[deprecated("Pointer element types are deprecated. You can *temporarily* " - "use Type::getPointerElementType() instead")]] - Type *getElementType() const { - assert(!isOpaque() && "Attempting to get element type of opaque pointer"); - return PointeeTy; - } - bool isOpaque() const { return !PointeeTy; } /// Return true if the specified type is valid as a element type. diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h index 1ea1d9787d61..da37801b6d19 100644 --- a/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/llvm/include/llvm/IR/DiagnosticInfo.h @@ -85,6 +85,7 @@ enum DiagnosticKind { DK_Unsupported, DK_SrcMgr, DK_DontCall, + DK_MisExpect, DK_FirstPluginKind // Must be last value to work with // getNextAvailablePluginDiagnosticKind }; @@ -1032,6 +1033,25 @@ public: void print(DiagnosticPrinter &DP) const override; }; +/// Diagnostic information for MisExpect analysis. +class DiagnosticInfoMisExpect : public DiagnosticInfoWithLocationBase { +public: + DiagnosticInfoMisExpect(const Instruction *Inst, Twine &Msg); + + /// \see DiagnosticInfo::print. + void print(DiagnosticPrinter &DP) const override; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_MisExpect; + } + + const Twine &getMsg() const { return Msg; } + +private: + /// Message to report. + const Twine &Msg; +}; + static DiagnosticSeverity getDiagnosticSeverity(SourceMgr::DiagKind DK) { switch (DK) { case llvm::SourceMgr::DK_Error: diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index d13a5856df3b..a381c075d77b 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -14,6 +14,7 @@ #ifndef LLVM_IR_DOMINATORS_H #define LLVM_IR_DOMINATORS_H +#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" @@ -22,6 +23,8 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" +#include "llvm/ADT/ilist_iterator.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/PassManager.h" @@ -31,6 +34,7 @@ #include "llvm/Support/CFGUpdate.h" #include "llvm/Support/GenericDomTree.h" #include "llvm/Support/GenericDomTreeConstruction.h" +#include <algorithm> #include <utility> #include <vector> diff --git a/llvm/include/llvm/IR/FMF.h b/llvm/include/llvm/IR/FMF.h new file mode 100644 index 000000000000..a49feb5a8946 --- /dev/null +++ b/llvm/include/llvm/IR/FMF.h @@ -0,0 +1,121 @@ +//===-- llvm/FMF.h - Fast math flags subclass -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the fast math flags. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_FMF_H +#define LLVM_IR_FMF_H + +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +/// Convenience struct for specifying and reasoning about fast-math flags. +class FastMathFlags { +private: + friend class FPMathOperator; + + unsigned Flags = 0; + + FastMathFlags(unsigned F) { + // If all 7 bits are set, turn this into -1. If the number of bits grows, + // this must be updated. This is intended to provide some forward binary + // compatibility insurance for the meaning of 'fast' in case bits are added. + if (F == 0x7F) Flags = ~0U; + else Flags = F; + } + +public: + // This is how the bits are used in Value::SubclassOptionalData so they + // should fit there too. + // WARNING: We're out of space. SubclassOptionalData only has 7 bits. New + // functionality will require a change in how this information is stored. + enum { + AllowReassoc = (1 << 0), + NoNaNs = (1 << 1), + NoInfs = (1 << 2), + NoSignedZeros = (1 << 3), + AllowReciprocal = (1 << 4), + AllowContract = (1 << 5), + ApproxFunc = (1 << 6) + }; + + FastMathFlags() = default; + + static FastMathFlags getFast() { + FastMathFlags FMF; + FMF.setFast(); + return FMF; + } + + bool any() const { return Flags != 0; } + bool none() const { return Flags == 0; } + bool all() const { return Flags == ~0U; } + + void clear() { Flags = 0; } + void set() { Flags = ~0U; } + + /// Flag queries + bool allowReassoc() const { return 0 != (Flags & AllowReassoc); } + bool noNaNs() const { return 0 != (Flags & NoNaNs); } + bool noInfs() const { return 0 != (Flags & NoInfs); } + bool noSignedZeros() const { return 0 != (Flags & NoSignedZeros); } + bool allowReciprocal() const { return 0 != (Flags & AllowReciprocal); } + bool allowContract() const { return 0 != (Flags & AllowContract); } + bool approxFunc() const { return 0 != (Flags & ApproxFunc); } + /// 'Fast' means all bits are set. + bool isFast() const { return all(); } + + /// Flag setters + void setAllowReassoc(bool B = true) { + Flags = (Flags & ~AllowReassoc) | B * AllowReassoc; + } + void setNoNaNs(bool B = true) { + Flags = (Flags & ~NoNaNs) | B * NoNaNs; + } + void setNoInfs(bool B = true) { + Flags = (Flags & ~NoInfs) | B * NoInfs; + } + void setNoSignedZeros(bool B = true) { + Flags = (Flags & ~NoSignedZeros) | B * NoSignedZeros; + } + void setAllowReciprocal(bool B = true) { + Flags = (Flags & ~AllowReciprocal) | B * AllowReciprocal; + } + void setAllowContract(bool B = true) { + Flags = (Flags & ~AllowContract) | B * AllowContract; + } + void setApproxFunc(bool B = true) { + Flags = (Flags & ~ApproxFunc) | B * ApproxFunc; + } + void setFast(bool B = true) { B ? set() : clear(); } + + void operator&=(const FastMathFlags &OtherFlags) { + Flags &= OtherFlags.Flags; + } + void operator|=(const FastMathFlags &OtherFlags) { + Flags |= OtherFlags.Flags; + } + bool operator!=(const FastMathFlags &OtherFlags) const { + return Flags != OtherFlags.Flags; + } + + /// Print fast-math flags to \p O. + void print(raw_ostream &O) const; +}; + +inline raw_ostream &operator<<(raw_ostream &O, FastMathFlags FMF) { + FMF.print(O); + return O; +} + +} // end namespace llvm + +#endif // LLVM_IR_FMF_H diff --git a/llvm/include/llvm/IR/FPEnv.h b/llvm/include/llvm/IR/FPEnv.h index bf435ec6d109..e598db224211 100644 --- a/llvm/include/llvm/IR/FPEnv.h +++ b/llvm/include/llvm/IR/FPEnv.h @@ -17,10 +17,17 @@ #include "llvm/ADT/FloatingPointMode.h" #include "llvm/ADT/Optional.h" +#include "llvm/IR/FMF.h" namespace llvm { class StringRef; +namespace Intrinsic { +typedef unsigned ID; +} + +class Instruction; + namespace fp { /// Exception behavior used for floating point operations. @@ -59,10 +66,22 @@ inline bool isDefaultFPEnvironment(fp::ExceptionBehavior EB, RoundingMode RM) { return EB == fp::ebIgnore && RM == RoundingMode::NearestTiesToEven; } +/// Returns constrained intrinsic id to represent the given instruction in +/// strictfp function. If the instruction is already a constrained intrinsic or +/// does not have a constrained intrinsic counterpart, the function returns +/// zero. +Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr); + /// Returns true if the rounding mode RM may be QRM at compile time or /// at run time. inline bool canRoundingModeBe(RoundingMode RM, RoundingMode QRM) { return RM == QRM || RM == RoundingMode::Dynamic; } + +/// Returns true if the possibility of a signaling NaN can be safely +/// ignored. +inline bool canIgnoreSNaN(fp::ExceptionBehavior EB, FastMathFlags FMF) { + return (EB == fp::ebIgnore || FMF.noNaNs()); +} } #endif diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index 31979cd2f9db..7c32c5d13760 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -42,3 +42,5 @@ LLVM_FIXED_MD_KIND(MD_preserve_access_index, "llvm.preserve.access.index", 27) LLVM_FIXED_MD_KIND(MD_vcall_visibility, "vcall_visibility", 28) LLVM_FIXED_MD_KIND(MD_noundef, "noundef", 29) LLVM_FIXED_MD_KIND(MD_annotation, "annotation", 30) +LLVM_FIXED_MD_KIND(MD_nosanitize, "nosanitize", 31) +LLVM_FIXED_MD_KIND(MD_func_sanitize, "func_sanitize", 32) diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index 90095cd1bc77..7945c64c8610 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -32,8 +32,6 @@ #include "llvm/IR/OperandTraits.h" #include "llvm/IR/SymbolTableListTraits.h" #include "llvm/IR/Value.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Compiler.h" #include <cassert> #include <cstddef> #include <cstdint> @@ -290,7 +288,7 @@ public: /// profile annotations. If IncludeSynthetic is false, only return true /// when the profile data is real. bool hasProfileData(bool IncludeSynthetic = false) const { - return getEntryCount(IncludeSynthetic).hasValue(); + return getEntryCount(IncludeSynthetic).has_value(); } /// Returns the set of GUIDs that needs to be imported to the function for @@ -486,11 +484,12 @@ public: return AttributeSets.getParamDereferenceableOrNullBytes(ArgNo); } - /// A function will have the "coroutine.presplit" attribute if it's - /// a coroutine and has not gone through full CoroSplit pass. + /// Determine if the function is presplit coroutine. bool isPresplitCoroutine() const { - return hasFnAttribute("coroutine.presplit"); + return hasFnAttribute(Attribute::PresplitCoroutine); } + void setPresplitCoroutine() { addFnAttr(Attribute::PresplitCoroutine); } + void setSplittedCoroutine() { removeFnAttr(Attribute::PresplitCoroutine); } /// Determine if the function does not access memory. bool doesNotAccessMemory() const { @@ -623,15 +622,19 @@ public: bool willReturn() const { return hasFnAttribute(Attribute::WillReturn); } void setWillReturn() { addFnAttr(Attribute::WillReturn); } + /// Get what kind of unwind table entry to generate for this function. + UWTableKind getUWTableKind() const { + return AttributeSets.getUWTableKind(); + } + /// True if the ABI mandates (or the user requested) that this /// function be in a unwind table. bool hasUWTable() const { - return hasFnAttribute(Attribute::UWTable); + return getUWTableKind() != UWTableKind::None; } - void setHasUWTable() { - addFnAttr(Attribute::UWTable); + void setUWTableKind(UWTableKind K) { + addFnAttr(Attribute::getWithUWTableKind(getContext(), K)); } - /// True if this function needs an unwind table. bool needsUnwindTableEntry() const { return hasUWTable() || !doesNotThrow() || hasPersonalityFn(); diff --git a/llvm/include/llvm/IR/GCStrategy.h b/llvm/include/llvm/IR/GCStrategy.h index 4fa8e3a8dcf4..41024469044f 100644 --- a/llvm/include/llvm/IR/GCStrategy.h +++ b/llvm/include/llvm/IR/GCStrategy.h @@ -38,9 +38,7 @@ // When used with gc.statepoint, information about safepoint and roots can be // found in the binary StackMap section after code generation. Safepoint // placement is currently the responsibility of the frontend, though late -// insertion support is planned. gc.statepoint does not currently support -// custom stack map formats; such can be generated by parsing the standard -// stack map section if desired. +// insertion support is planned. // // The read and write barrier support can be used with either implementation. // @@ -101,6 +99,11 @@ public: } ///@} + /// If set, appropriate metadata tables must be emitted by the back-end + /// (assembler, JIT, or otherwise). The default stackmap information can be + /// found in the StackMap section as described in the documentation. + bool usesMetadata() const { return UsesMetadata; } + /** @name GCRoot Specific Properties * These properties and overrides only apply to collector strategies using * GCRoot. @@ -110,12 +113,6 @@ public: /// True if safe points need to be inferred on call sites bool needsSafePoints() const { return NeededSafePoints; } - /// If set, appropriate metadata tables must be emitted by the back-end - /// (assembler, JIT, or otherwise). For statepoint, this method is - /// currently unsupported. The stackmap information can be found in the - /// StackMap section as described in the documentation. - bool usesMetadata() const { return UsesMetadata; } - ///@} }; @@ -126,7 +123,7 @@ public: /// static GCRegistry::Add<CustomGC> X("custom-name", /// "my custom supper fancy gc strategy"); /// -/// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also +/// Note that to use a custom GCMetadataPrinter, you must also /// register your GCMetadataPrinter subclass with the /// GCMetadataPrinterRegistery as well. using GCRegistry = Registry<GCStrategy>; diff --git a/llvm/include/llvm/IR/GlobalIFunc.h b/llvm/include/llvm/IR/GlobalIFunc.h index 10088ee2fff4..976772b343fd 100644 --- a/llvm/include/llvm/IR/GlobalIFunc.h +++ b/llvm/include/llvm/IR/GlobalIFunc.h @@ -84,6 +84,11 @@ public: return FunctionType::get(IFuncValTy->getPointerTo(), false); } + static bool isValidLinkage(LinkageTypes L) { + return isExternalLinkage(L) || isLocalLinkage(L) || isWeakLinkage(L) || + isLinkOnceLinkage(L); + } + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { return V->getValueID() == Value::GlobalIFuncVal; diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h index 0bb9fd730059..96a270316686 100644 --- a/llvm/include/llvm/IR/GlobalObject.h +++ b/llvm/include/llvm/IR/GlobalObject.h @@ -43,13 +43,12 @@ protected: GlobalObject(Type *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, LinkageTypes Linkage, const Twine &Name, unsigned AddressSpace = 0) - : GlobalValue(Ty, VTy, Ops, NumOps, Linkage, Name, AddressSpace), - ObjComdat(nullptr) { + : GlobalValue(Ty, VTy, Ops, NumOps, Linkage, Name, AddressSpace) { setGlobalValueSubClassData(0); } ~GlobalObject(); - Comdat *ObjComdat; + Comdat *ObjComdat = nullptr; enum { LastAlignmentBit = 5, HasSectionHashEntryBit, diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index 1818f2a8f3cc..a17423dd965b 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -80,14 +80,14 @@ protected: UnnamedAddrVal(unsigned(UnnamedAddr::None)), DllStorageClass(DefaultStorageClass), ThreadLocal(NotThreadLocal), HasLLVMReservedName(false), IsDSOLocal(false), HasPartition(false), - IntID((Intrinsic::ID)0U), Parent(nullptr) { + HasSanitizerMetadata(false) { setLinkage(Linkage); setName(Name); } Type *ValueType; - static const unsigned GlobalValueSubClassDataBits = 16; + static const unsigned GlobalValueSubClassDataBits = 15; // All bitfields use unsigned as the underlying type so that MSVC will pack // them. @@ -112,9 +112,14 @@ protected: /// https://lld.llvm.org/Partitions.html). unsigned HasPartition : 1; + /// True if this symbol has sanitizer metadata available. Should only happen + /// if sanitizers were enabled when building the translation unit which + /// contains this GV. + unsigned HasSanitizerMetadata : 1; + private: // Give subclasses access to what otherwise would be wasted padding. - // (16 + 4 + 2 + 2 + 2 + 3 + 1 + 1 + 1) == 32. + // (15 + 4 + 2 + 2 + 2 + 3 + 1 + 1 + 1 + 1) == 32. unsigned SubClassData : GlobalValueSubClassDataBits; friend class Constant; @@ -153,7 +158,7 @@ protected: /// Subclasses can use it to store their intrinsic ID, if they have one. /// /// This is stored here to save space in Function on 64-bit hosts. - Intrinsic::ID IntID; + Intrinsic::ID IntID = (Intrinsic::ID)0U; unsigned getGlobalValueSubClassData() const { return SubClassData; @@ -163,7 +168,7 @@ protected: SubClassData = V; } - Module *Parent; // The containing module. + Module *Parent = nullptr; // The containing module. // Used by SymbolTableListTraits. void setParent(Module *parent) { @@ -289,6 +294,43 @@ public: StringRef getPartition() const; void setPartition(StringRef Part); + // ASan, HWASan and Memtag sanitizers have some instrumentation that applies + // specifically to global variables. This instrumentation is implicitly + // applied to all global variables when built with -fsanitize=*. What we need + // is a way to persist the information that a certain global variable should + // *not* have sanitizers applied, which occurs if: + // 1. The global variable is in the sanitizer ignore list, or + // 2. The global variable is created by the sanitizers itself for internal + // usage, or + // 3. The global variable has __attribute__((no_sanitize("..."))) or + // __attribute__((disable_sanitizer_instrumentation)). + // + // This is important, a some IR passes like GlobalMerge can delete global + // variables and replace them with new ones. If the old variables were marked + // to be unsanitized, then the new ones should also be. + struct SanitizerMetadata { + SanitizerMetadata() + : NoAddress(false), NoHWAddress(false), NoMemtag(false), + IsDynInit(false) {} + unsigned NoAddress : 1; + unsigned NoHWAddress : 1; + unsigned NoMemtag : 1; + + // ASan-specific metadata. Is this global variable dynamically initialized + // (from a C++ language perspective), and should therefore be checked for + // ODR violations. + unsigned IsDynInit : 1; + }; + + bool hasSanitizerMetadata() const { return HasSanitizerMetadata; } + const SanitizerMetadata &getSanitizerMetadata() const; + // Note: Not byref as it's a POD and otherwise it's too easy to call + // G.setSanitizerMetadata(G2.getSanitizerMetadata()), and the argument becomes + // dangling when the backing storage allocates the metadata for `G`, as the + // storage is shared between `G1` and `G2`. + void setSanitizerMetadata(SanitizerMetadata Meta); + void removeSanitizerMetadata(); + static LinkageTypes getLinkOnceLinkage(bool ODR) { return ODR ? LinkOnceODRLinkage : LinkOnceAnyLinkage; } diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index a1789759960d..d8f08934b3d6 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -25,7 +25,6 @@ #include "llvm/IR/ConstantFolder.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/FPEnv.h" @@ -77,7 +76,7 @@ class IRBuilderCallbackInserter : public IRBuilderDefaultInserter { std::function<void(Instruction *)> Callback; public: - virtual ~IRBuilderCallbackInserter(); + ~IRBuilderCallbackInserter() override; IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback) : Callback(std::move(Callback)) {} @@ -125,21 +124,18 @@ protected: MDNode *DefaultFPMathTag; FastMathFlags FMF; - bool IsFPConstrained; - fp::ExceptionBehavior DefaultConstrainedExcept; - RoundingMode DefaultConstrainedRounding; + bool IsFPConstrained = false; + fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebStrict; + RoundingMode DefaultConstrainedRounding = RoundingMode::Dynamic; ArrayRef<OperandBundleDef> DefaultOperandBundles; public: IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder, - const IRBuilderDefaultInserter &Inserter, - MDNode *FPMathTag, ArrayRef<OperandBundleDef> OpBundles) + const IRBuilderDefaultInserter &Inserter, MDNode *FPMathTag, + ArrayRef<OperandBundleDef> OpBundles) : Context(context), Folder(Folder), Inserter(Inserter), - DefaultFPMathTag(FPMathTag), IsFPConstrained(false), - DefaultConstrainedExcept(fp::ebStrict), - DefaultConstrainedRounding(RoundingMode::Dynamic), - DefaultOperandBundles(OpBundles) { + DefaultFPMathTag(FPMathTag), DefaultOperandBundles(OpBundles) { ClearInsertionPoint(); } @@ -218,23 +214,11 @@ public: } /// Get location information used by debugging information. - DebugLoc getCurrentDebugLocation() const { - for (auto &KV : MetadataToCopy) - if (KV.first == LLVMContext::MD_dbg) - return {cast<DILocation>(KV.second)}; - - return {}; - } + DebugLoc getCurrentDebugLocation() const; /// If this builder has a current debug location, set it on the /// specified instruction. - void SetInstDebugLocation(Instruction *I) const { - for (const auto &KV : MetadataToCopy) - if (KV.first == LLVMContext::MD_dbg) { - I->setDebugLoc(DebugLoc(KV.second)); - return; - } - } + void SetInstDebugLocation(Instruction *I) const; /// Add all entries in MetadataToCopy to \p I. void AddMetadataToInst(Instruction *I) const { @@ -316,7 +300,7 @@ public: void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) { #ifndef NDEBUG Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept); - assert(ExceptStr.hasValue() && "Garbage strict exception behavior!"); + assert(ExceptStr && "Garbage strict exception behavior!"); #endif DefaultConstrainedExcept = NewExcept; } @@ -325,7 +309,7 @@ public: void setDefaultConstrainedRounding(RoundingMode NewRounding) { #ifndef NDEBUG Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding); - assert(RoundingStr.hasValue() && "Garbage strict rounding mode!"); + assert(RoundingStr && "Garbage strict rounding mode!"); #endif DefaultConstrainedRounding = NewRounding; } @@ -556,6 +540,11 @@ public: return Type::getVoidTy(Context); } + /// Fetch the type representing a pointer. + PointerType *getPtrTy(unsigned AddrSpace = 0) { + return PointerType::get(Context, AddrSpace); + } + /// Fetch the type representing a pointer to an 8-bit integer value. PointerType *getInt8PtrTy(unsigned AddrSpace = 0) { return Type::getInt8PtrTy(Context, AddrSpace); @@ -589,6 +578,12 @@ public: MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr); + CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val, + Value *Size, bool IsVolatile = false, + MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); + /// Create and insert an element unordered-atomic memset of the region of /// memory starting at the given pointer to the given value. /// @@ -789,7 +784,7 @@ public: /// Create a call to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, - Value *ActualCallee, + FunctionCallee ActualCallee, ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, @@ -798,7 +793,7 @@ public: /// Create a call to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, - Value *ActualCallee, uint32_t Flags, + FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs, Optional<ArrayRef<Use>> DeoptArgs, @@ -809,7 +804,8 @@ public: /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be /// .get()'ed to get the Value pointer. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, - Value *ActualCallee, ArrayRef<Use> CallArgs, + FunctionCallee ActualCallee, + ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name = ""); @@ -818,7 +814,7 @@ public: /// start a new statepoint sequence. InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, - Value *ActualInvokee, BasicBlock *NormalDest, + FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name = ""); @@ -826,7 +822,7 @@ public: /// Create an invoke to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. InvokeInst *CreateGCStatepointInvoke( - uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs, Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, @@ -837,7 +833,7 @@ public: // get the Value *. InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, - Value *ActualInvokee, BasicBlock *NormalDest, + FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name = ""); @@ -918,18 +914,18 @@ public: Name); } - /// Create a call to the experimental.vector.extract intrinsic. + /// Create a call to the vector.extract intrinsic. CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name = "") { - return CreateIntrinsic(Intrinsic::experimental_vector_extract, + return CreateIntrinsic(Intrinsic::vector_extract, {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr, Name); } - /// Create a call to the experimental.vector.insert intrinsic. + /// Create a call to the vector.insert intrinsic. CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name = "") { - return CreateIntrinsic(Intrinsic::experimental_vector_insert, + return CreateIntrinsic(Intrinsic::vector_insert, {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx}, nullptr, Name); } @@ -1162,21 +1158,14 @@ private: return I; } - Value *foldConstant(Instruction::BinaryOps Opc, Value *L, - Value *R, const Twine &Name) const { - auto *LC = dyn_cast<Constant>(L); - auto *RC = dyn_cast<Constant>(R); - return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr; - } - Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) { RoundingMode UseRounding = DefaultConstrainedRounding; - if (Rounding.hasValue()) + if (Rounding) UseRounding = Rounding.getValue(); Optional<StringRef> RoundingStr = convertRoundingModeToStr(UseRounding); - assert(RoundingStr.hasValue() && "Garbage strict rounding mode!"); + assert(RoundingStr && "Garbage strict rounding mode!"); auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue()); return MetadataAsValue::get(Context, RoundingMDS); @@ -1185,11 +1174,11 @@ private: Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) { fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept; - if (Except.hasValue()) + if (Except) UseExcept = Except.getValue(); Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(UseExcept); - assert(ExceptStr.hasValue() && "Garbage strict exception behavior!"); + assert(ExceptStr && "Garbage strict exception behavior!"); auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue()); return MetadataAsValue::get(Context, ExceptMDS); @@ -1210,10 +1199,11 @@ private: public: Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { - if (auto *V = Folder.FoldAdd(LHS, RHS, HasNUW, HasNSW)) + if (Value *V = + Folder.FoldNoWrapBinOp(Instruction::Add, LHS, RHS, HasNUW, HasNSW)) return V; - return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, - HasNUW, HasNSW); + return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, HasNUW, + HasNSW); } Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") { @@ -1226,11 +1216,11 @@ public: Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name); - return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, - HasNUW, HasNSW); + if (Value *V = + Folder.FoldNoWrapBinOp(Instruction::Sub, LHS, RHS, HasNUW, HasNSW)) + return V; + return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, HasNUW, + HasNSW); } Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") { @@ -1243,11 +1233,11 @@ public: Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name); - return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, - HasNUW, HasNSW); + if (Value *V = + Folder.FoldNoWrapBinOp(Instruction::Mul, LHS, RHS, HasNUW, HasNSW)) + return V; + return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, HasNUW, + HasNSW); } Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") { @@ -1260,9 +1250,8 @@ public: Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "", bool isExact = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateUDiv(LC, RC, isExact), Name); + if (Value *V = Folder.FoldExactBinOp(Instruction::UDiv, LHS, RHS, isExact)) + return V; if (!isExact) return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name); return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name); @@ -1274,9 +1263,8 @@ public: Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "", bool isExact = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateSDiv(LC, RC, isExact), Name); + if (Value *V = Folder.FoldExactBinOp(Instruction::SDiv, LHS, RHS, isExact)) + return V; if (!isExact) return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name); return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name); @@ -1287,20 +1275,22 @@ public: } Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") { - if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V; + if (Value *V = Folder.FoldBinOp(Instruction::URem, LHS, RHS)) + return V; return Insert(BinaryOperator::CreateURem(LHS, RHS), Name); } Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") { - if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V; + if (Value *V = Folder.FoldBinOp(Instruction::SRem, LHS, RHS)) + return V; return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name); } Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name); + if (Value *V = + Folder.FoldNoWrapBinOp(Instruction::Shl, LHS, RHS, HasNUW, HasNSW)) + return V; return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name, HasNUW, HasNSW); } @@ -1319,9 +1309,8 @@ public: Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "", bool isExact = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateLShr(LC, RC, isExact), Name); + if (Value *V = Folder.FoldExactBinOp(Instruction::LShr, LHS, RHS, isExact)) + return V; if (!isExact) return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name); return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name); @@ -1339,9 +1328,8 @@ public: Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "", bool isExact = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateAShr(LC, RC, isExact), Name); + if (Value *V = Folder.FoldExactBinOp(Instruction::AShr, LHS, RHS, isExact)) + return V; if (!isExact) return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name); return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name); @@ -1358,7 +1346,7 @@ public: } Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") { - if (auto *V = Folder.FoldAnd(LHS, RHS)) + if (auto *V = Folder.FoldBinOp(Instruction::And, LHS, RHS)) return V; return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name); } @@ -1380,7 +1368,7 @@ public: } Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") { - if (auto *V = Folder.FoldOr(LHS, RHS)) + if (auto *V = Folder.FoldBinOp(Instruction::Or, LHS, RHS)) return V; return Insert(BinaryOperator::CreateOr(LHS, RHS), Name); } @@ -1402,7 +1390,8 @@ public: } Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") { - if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V; + if (Value *V = Folder.FoldBinOp(Instruction::Xor, LHS, RHS)) + return V; return Insert(BinaryOperator::CreateXor(LHS, RHS), Name); } @@ -1420,7 +1409,8 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, L, R, nullptr, Name, FPMD); - if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; + if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF)) + return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF); return Insert(I, Name); } @@ -1433,9 +1423,10 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, L, R, FMFSource, Name); - if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; - Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, - FMFSource->getFastMathFlags()); + FastMathFlags FMF = FMFSource->getFastMathFlags(); + if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF)) + return V; + Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, FMF); return Insert(I, Name); } @@ -1445,7 +1436,8 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, L, R, nullptr, Name, FPMD); - if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; + if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF)) + return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF); return Insert(I, Name); } @@ -1458,9 +1450,10 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, L, R, FMFSource, Name); - if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; - Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, - FMFSource->getFastMathFlags()); + FastMathFlags FMF = FMFSource->getFastMathFlags(); + if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF)) + return V; + Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, FMF); return Insert(I, Name); } @@ -1470,7 +1463,8 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, L, R, nullptr, Name, FPMD); - if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; + if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF)) + return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF); return Insert(I, Name); } @@ -1483,9 +1477,10 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, L, R, FMFSource, Name); - if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; - Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, - FMFSource->getFastMathFlags()); + FastMathFlags FMF = FMFSource->getFastMathFlags(); + if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF)) + return V; + Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, FMF); return Insert(I, Name); } @@ -1495,7 +1490,8 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, L, R, nullptr, Name, FPMD); - if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; + if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF)) + return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF); return Insert(I, Name); } @@ -1508,9 +1504,9 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, L, R, FMFSource, Name); - if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; - Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, - FMFSource->getFastMathFlags()); + if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF)) + return V; + Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, FMF); return Insert(I, Name); } @@ -1520,7 +1516,7 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, L, R, nullptr, Name, FPMD); - if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; + if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF); return Insert(I, Name); } @@ -1533,16 +1529,16 @@ public: return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, L, R, FMFSource, Name); - if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; - Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, - FMFSource->getFastMathFlags()); + FastMathFlags FMF = FMFSource->getFastMathFlags(); + if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V; + Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, FMF); return Insert(I, Name); } Value *CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name = "", MDNode *FPMathTag = nullptr) { - if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V; + if (Value *V = Folder.FoldBinOp(Opc, LHS, RHS)) return V; Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS); if (isa<FPMathOperator>(BinOp)) setFPAttrs(BinOp, FPMathTag, FMF); @@ -1576,14 +1572,10 @@ public: Optional<RoundingMode> Rounding = None, Optional<fp::ExceptionBehavior> Except = None); - Value *CreateNeg(Value *V, const Twine &Name = "", - bool HasNUW = false, bool HasNSW = false) { - if (auto *VC = dyn_cast<Constant>(V)) - return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name); - BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; + Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNUW = false, + bool HasNSW = false) { + return CreateSub(Constant::getNullValue(V->getType()), V, Name, HasNUW, + HasNSW); } Value *CreateNSWNeg(Value *V, const Twine &Name = "") { @@ -1614,9 +1606,7 @@ public: } Value *CreateNot(Value *V, const Twine &Name = "") { - if (auto *VC = dyn_cast<Constant>(V)) - return Insert(Folder.CreateNot(VC), Name); - return Insert(BinaryOperator::CreateNot(V), Name); + return CreateXor(V, Constant::getAllOnesValue(V->getType()), Name); } Value *CreateUnOp(Instruction::UnaryOps Opc, @@ -1733,30 +1723,18 @@ public: } Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, - const Twine &Name = "") { - if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/false)) + const Twine &Name = "", bool IsInBounds = false) { + if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, IsInBounds)) return V; - return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name); + return Insert(IsInBounds + ? GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList) + : GetElementPtrInst::Create(Ty, Ptr, IdxList), + Name); } Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, const Twine &Name = "") { - if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/true)) - return V; - return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name); - } - - Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") { - if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/false)) - return V; - return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); - } - - Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx, - const Twine &Name = "") { - if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/true)) - return V; - return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); + return CreateGEP(Ty, Ptr, IdxList, Name, /* IsInBounds */ true); } Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, @@ -2297,9 +2275,8 @@ public: Value *CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name = "") { - if (auto *VC = dyn_cast<Constant>(Vec)) - if (auto *IC = dyn_cast<Constant>(Idx)) - return Insert(Folder.CreateExtractElement(VC, IC), Name); + if (Value *V = Folder.FoldExtractElement(Vec, Idx)) + return V; return Insert(ExtractElementInst::Create(Vec, Idx), Name); } @@ -2320,10 +2297,8 @@ public: Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, const Twine &Name = "") { - if (auto *VC = dyn_cast<Constant>(Vec)) - if (auto *NC = dyn_cast<Constant>(NewElt)) - if (auto *IC = dyn_cast<Constant>(Idx)) - return Insert(Folder.CreateInsertElement(VC, NC, IC), Name); + if (Value *V = Folder.FoldInsertElement(Vec, NewElt, Idx)) + return V; return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name); } @@ -2339,21 +2314,11 @@ public: return CreateShuffleVector(V1, V2, IntMask, Name); } - LLVM_ATTRIBUTE_DEPRECATED(Value *CreateShuffleVector(Value *V1, Value *V2, - ArrayRef<uint32_t> Mask, - const Twine &Name = ""), - "Pass indices as 'int' instead") { - SmallVector<int, 16> IntMask; - IntMask.assign(Mask.begin(), Mask.end()); - return CreateShuffleVector(V1, V2, IntMask, Name); - } - /// See class ShuffleVectorInst for a description of the mask representation. Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask, const Twine &Name = "") { - if (auto *V1C = dyn_cast<Constant>(V1)) - if (auto *V2C = dyn_cast<Constant>(V2)) - return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name); + if (Value *V = Folder.FoldShuffleVector(V1, V2, Mask)) + return V; return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); } @@ -2364,20 +2329,17 @@ public: return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name); } - Value *CreateExtractValue(Value *Agg, - ArrayRef<unsigned> Idxs, + Value *CreateExtractValue(Value *Agg, ArrayRef<unsigned> Idxs, const Twine &Name = "") { - if (auto *AggC = dyn_cast<Constant>(Agg)) - return Insert(Folder.CreateExtractValue(AggC, Idxs), Name); + if (auto *V = Folder.FoldExtractValue(Agg, Idxs)) + return V; return Insert(ExtractValueInst::Create(Agg, Idxs), Name); } - Value *CreateInsertValue(Value *Agg, Value *Val, - ArrayRef<unsigned> Idxs, + Value *CreateInsertValue(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, const Twine &Name = "") { - if (auto *AggC = dyn_cast<Constant>(Agg)) - if (auto *ValC = dyn_cast<Constant>(Val)) - return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name); + if (auto *V = Folder.FoldInsertValue(Agg, Val, Idxs)) + return V; return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name); } @@ -2394,16 +2356,25 @@ public: // Utility creation methods //===--------------------------------------------------------------------===// - /// Return an i1 value testing if \p Arg is null. + /// Return a boolean value testing if \p Arg == 0. Value *CreateIsNull(Value *Arg, const Twine &Name = "") { - return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()), - Name); + return CreateICmpEQ(Arg, ConstantInt::getNullValue(Arg->getType()), Name); } - /// Return an i1 value testing if \p Arg is not null. + /// Return a boolean value testing if \p Arg != 0. Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") { - return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()), - Name); + return CreateICmpNE(Arg, ConstantInt::getNullValue(Arg->getType()), Name); + } + + /// Return a boolean value testing if \p Arg < 0. + Value *CreateIsNeg(Value *Arg, const Twine &Name = "") { + return CreateICmpSLT(Arg, ConstantInt::getNullValue(Arg->getType()), Name); + } + + /// Return a boolean value testing if \p Arg > -1. + Value *CreateIsNotNeg(Value *Arg, const Twine &Name = "") { + return CreateICmpSGT(Arg, ConstantInt::getAllOnesValue(Arg->getType()), + Name); } /// Return the i64 difference between two pointer values, dividing out diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h index 2827ab553adc..9505f1e3be2a 100644 --- a/llvm/include/llvm/IR/IRBuilderFolder.h +++ b/llvm/include/llvm/IR/IRBuilderFolder.h @@ -31,12 +31,19 @@ public: // Return an existing value or a constant if the operation can be simplified. // Otherwise return nullptr. //===--------------------------------------------------------------------===// - virtual Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, - bool HasNSW = false) const = 0; - virtual Value *FoldAnd(Value *LHS, Value *RHS) const = 0; + virtual Value *FoldBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS) const = 0; - virtual Value *FoldOr(Value *LHS, Value *RHS) const = 0; + virtual Value *FoldExactBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS, bool IsExact) const = 0; + + virtual Value *FoldNoWrapBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS, bool HasNUW, + bool HasNSW) const = 0; + + virtual Value *FoldBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS, FastMathFlags FMF) const = 0; virtual Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const = 0; @@ -46,43 +53,25 @@ public: virtual Value *FoldSelect(Value *C, Value *True, Value *False) const = 0; - //===--------------------------------------------------------------------===// - // Binary Operators - //===--------------------------------------------------------------------===// + virtual Value *FoldExtractValue(Value *Agg, + ArrayRef<unsigned> IdxList) const = 0; + + virtual Value *FoldInsertValue(Value *Agg, Value *Val, + ArrayRef<unsigned> IdxList) const = 0; + + virtual Value *FoldExtractElement(Value *Vec, Value *Idx) const = 0; - virtual Value *CreateFAdd(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateSub(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const = 0; - virtual Value *CreateFSub(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateMul(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const = 0; - virtual Value *CreateFMul(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateUDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const = 0; - virtual Value *CreateSDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const = 0; - virtual Value *CreateFDiv(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateURem(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateSRem(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateFRem(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateShl(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const = 0; - virtual Value *CreateLShr(Constant *LHS, Constant *RHS, - bool isExact = false) const = 0; - virtual Value *CreateAShr(Constant *LHS, Constant *RHS, - bool isExact = false) const = 0; - virtual Value *CreateXor(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateBinOp(Instruction::BinaryOps Opc, - Constant *LHS, Constant *RHS) const = 0; + virtual Value *FoldInsertElement(Value *Vec, Value *NewElt, + Value *Idx) const = 0; + + virtual Value *FoldShuffleVector(Value *V1, Value *V2, + ArrayRef<int> Mask) const = 0; //===--------------------------------------------------------------------===// // Unary Operators //===--------------------------------------------------------------------===// - virtual Value *CreateNeg(Constant *C, - bool HasNUW = false, bool HasNSW = false) const = 0; virtual Value *CreateFNeg(Constant *C) const = 0; - virtual Value *CreateNot(Constant *C) const = 0; virtual Value *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const = 0; //===--------------------------------------------------------------------===// @@ -110,20 +99,6 @@ public: virtual Value *CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const = 0; - - //===--------------------------------------------------------------------===// - // Other Instructions - //===--------------------------------------------------------------------===// - - virtual Value *CreateExtractElement(Constant *Vec, Constant *Idx) const = 0; - virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt, - Constant *Idx) const = 0; - virtual Value *CreateShuffleVector(Constant *V1, Constant *V2, - ArrayRef<int> Mask) const = 0; - virtual Value *CreateExtractValue(Constant *Agg, - ArrayRef<unsigned> IdxList) const = 0; - virtual Value *CreateInsertValue(Constant *Agg, Constant *Val, - ArrayRef<unsigned> IdxList) const = 0; }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/InlineAsm.h b/llvm/include/llvm/IR/InlineAsm.h index cf6b7af96980..57f2da27e04e 100644 --- a/llvm/include/llvm/IR/InlineAsm.h +++ b/llvm/include/llvm/IR/InlineAsm.h @@ -240,12 +240,15 @@ public: Kind_RegDefEarlyClobber = 3, // Early-clobber output register, "=&r". Kind_Clobber = 4, // Clobbered register, "~r". Kind_Imm = 5, // Immediate. - Kind_Mem = 6, // Memory operand, "m". + Kind_Mem = 6, // Memory operand, "m", or an address, "p". // Memory constraint codes. // These could be tablegenerated but there's little need to do that since // there's plenty of space in the encoding to support the union of all // constraint codes for all targets. + // Addresses are included here as they need to be treated the same by the + // backend, the only difference is that they are not used to actaully + // access memory by the instruction. Constraint_Unknown = 0, Constraint_es, Constraint_i, @@ -268,7 +271,15 @@ public: Constraint_Z, Constraint_ZC, Constraint_Zy, - Constraints_Max = Constraint_Zy, + + // Address constraints + Constraint_p, + Constraint_ZQ, + Constraint_ZR, + Constraint_ZS, + Constraint_ZT, + + Constraints_Max = Constraint_ZT, Constraints_ShiftAmount = 16, Flag_MatchingOperand = 0x80000000 @@ -453,6 +464,16 @@ public: return "ZC"; case InlineAsm::Constraint_Zy: return "Zy"; + case InlineAsm::Constraint_p: + return "p"; + case InlineAsm::Constraint_ZQ: + return "ZQ"; + case InlineAsm::Constraint_ZR: + return "ZR"; + case InlineAsm::Constraint_ZS: + return "ZS"; + case InlineAsm::Constraint_ZT: + return "ZT"; default: llvm_unreachable("Unknown memory constraint"); } diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h index 585129904dd4..7fec081d8155 100644 --- a/llvm/include/llvm/IR/InstVisitor.h +++ b/llvm/include/llvm/IR/InstVisitor.h @@ -15,7 +15,6 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" -#include "llvm/Support/ErrorHandling.h" namespace llvm { @@ -200,7 +199,7 @@ public: RetTy visitCatchPadInst(CatchPadInst &I) { DELEGATE(FuncletPadInst); } RetTy visitFreezeInst(FreezeInst &I) { DELEGATE(Instruction); } - // Handle the special instrinsic instruction classes. + // Handle the special intrinsic instruction classes. RetTy visitDbgDeclareInst(DbgDeclareInst &I) { DELEGATE(DbgVariableIntrinsic);} RetTy visitDbgValueInst(DbgValueInst &I) { DELEGATE(DbgVariableIntrinsic);} RetTy visitDbgVariableIntrinsic(DbgVariableIntrinsic &I) diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 589926c0faf1..eb6f89d740c6 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -21,22 +21,16 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Sequence.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/OperandTraits.h" -#include "llvm/IR/Type.h" #include "llvm/IR/User.h" -#include "llvm/IR/Value.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -47,6 +41,10 @@ namespace llvm { +class StringRef; +class Type; +class Value; + namespace Intrinsic { typedef unsigned ID; } @@ -1615,12 +1613,18 @@ public: /// Get the attribute of a given kind for the function. Attribute getFnAttr(StringRef Kind) const { - return getAttributes().getFnAttr(Kind); + Attribute Attr = getAttributes().getFnAttr(Kind); + if (Attr.isValid()) + return Attr; + return getFnAttrOnCalledFunction(Kind); } /// Get the attribute of a given kind for the function. Attribute getFnAttr(Attribute::AttrKind Kind) const { - return getAttributes().getFnAttr(Kind); + Attribute A = getAttributes().getFnAttr(Kind); + if (A.isValid()) + return A; + return getFnAttrOnCalledFunction(Kind); } /// Get the attribute of a given kind from a given arg @@ -1761,7 +1765,7 @@ public: return nullptr; } - /// Extract the preallocated type for a call or parameter. + /// Extract the inalloca type for a call or parameter. Type *getParamInAllocaType(unsigned ArgNo) const { if (auto *Ty = Attrs.getParamInAllocaType(ArgNo)) return Ty; @@ -1770,6 +1774,22 @@ public: return nullptr; } + /// Extract the sret type for a call or parameter. + Type *getParamStructRetType(unsigned ArgNo) const { + if (auto *Ty = Attrs.getParamStructRetType(ArgNo)) + return Ty; + if (const Function *F = getCalledFunction()) + return F->getAttributes().getParamStructRetType(ArgNo); + return nullptr; + } + + /// Extract the elementtype type for a parameter. + /// Note that elementtype() can only be applied to call arguments, not + /// function declaration parameters. + Type *getParamElementType(unsigned ArgNo) const { + return Attrs.getParamElementType(ArgNo); + } + /// Extract the number of dereferenceable bytes for a call or /// parameter (0=unknown). uint64_t getRetDereferenceableBytes() const { @@ -1806,7 +1826,13 @@ public: /// If one of the arguments has the 'returned' attribute, returns its /// operand value. Otherwise, return nullptr. - Value *getReturnedArgOperand() const; + Value *getReturnedArgOperand() const { + return getArgOperandWithAttribute(Attribute::Returned); + } + + /// If one of the arguments has the specified attribute, returns its + /// operand value. Otherwise, return nullptr. + Value *getArgOperandWithAttribute(Attribute::AttrKind Kind) const; /// Return true if the call should not be treated as a call to a /// builtin. @@ -2052,7 +2078,8 @@ public: bool hasClobberingOperandBundles() const { for (auto &BOI : bundle_op_infos()) { if (BOI.Tag->second == LLVMContext::OB_deopt || - BOI.Tag->second == LLVMContext::OB_funclet) + BOI.Tag->second == LLVMContext::OB_funclet || + BOI.Tag->second == LLVMContext::OB_ptrauth) continue; // This instruction has an operand bundle that is not known to us. @@ -2296,6 +2323,7 @@ private: return hasFnAttrOnCalledFunction(Kind); } + template <typename AK> Attribute getFnAttrOnCalledFunction(AK Kind) const; /// A specialized version of hasFnAttrImpl for when the caller wants to /// know if an attribute's semantics are implied, not whether the attribute diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h index 1937ffd36f7b..8d0a8363cdfb 100644 --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -24,7 +24,6 @@ #include "llvm/IR/User.h" #include "llvm/IR/Value.h" #include "llvm/Support/AtomicOrdering.h" -#include "llvm/Support/Casting.h" #include <cstdint> #include <utility> diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 5929cff3b4fb..d152e86488e1 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -21,24 +21,18 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/OperandTraits.h" -#include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/User.h" -#include "llvm/IR/Value.h" #include "llvm/Support/AtomicOrdering.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include <cassert> #include <cstddef> @@ -47,9 +41,14 @@ namespace llvm { +class APFloat; class APInt; +class BasicBlock; class ConstantInt; class DataLayout; +class StringRef; +class Type; +class Value; //===----------------------------------------------------------------------===// // AllocaInst Class @@ -127,9 +126,6 @@ public: setSubclassData<AlignmentField>(Log2(Align)); } - // FIXME: Remove this one transition to Align is over. - uint64_t getAlignment() const { return getAlign().value(); } - /// Return true if this alloca is in the entry block of the function and is a /// constant size. If so, the code generator will fold it into the /// prolog/epilog code, so it is basically free. @@ -217,11 +213,6 @@ public: void setVolatile(bool V) { setSubclassData<VolatileField>(V); } /// Return the alignment of the access that is being performed. - /// FIXME: Remove this function once transition to Align is over. - /// Use getAlign() instead. - uint64_t getAlignment() const { return getAlign().value(); } - - /// Return the alignment of the access that is being performed. Align getAlign() const { return Align(1ULL << (getSubclassData<AlignmentField>())); } @@ -347,11 +338,6 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// Return the alignment of the access that is being performed - /// FIXME: Remove this function once transition to Align is over. - /// Use getAlign() instead. - uint64_t getAlignment() const { return getAlign().value(); } - Align getAlign() const { return Align(1ULL << (getSubclassData<AlignmentField>())); } @@ -2138,6 +2124,12 @@ public: static bool isIdentityMask(ArrayRef<int> Mask); static bool isIdentityMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); + + // Not possible to express a shuffle mask for a scalable vector for this + // case. + if (isa<ScalableVectorType>(Mask->getType())) + return false; + SmallVector<int, 16> MaskAsInts; getShuffleMask(Mask, MaskAsInts); return isIdentityMask(MaskAsInts); @@ -2148,6 +2140,11 @@ public: /// from its input vectors. /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef> bool isIdentity() const { + // Not possible to express a shuffle mask for a scalable vector for this + // case. + if (isa<ScalableVectorType>(getType())) + return false; + return !changesLength() && isIdentityMask(ShuffleMask); } @@ -5311,6 +5308,10 @@ public: } }; +//===----------------------------------------------------------------------===// +// Helper functions +//===----------------------------------------------------------------------===// + /// A helper function that returns the pointer operand of a load or store /// instruction. Returns nullptr if not load or store. inline const Value *getLoadStorePointerOperand(const Value *V) { @@ -5366,6 +5367,24 @@ inline Type *getLoadStoreType(Value *I) { return cast<StoreInst>(I)->getValueOperand()->getType(); } +/// A helper function that returns an atomic operation's sync scope; returns +/// None if it is not an atomic operation. +inline Optional<SyncScope::ID> getAtomicSyncScopeID(const Instruction *I) { + if (!I->isAtomic()) + return None; + if (auto *AI = dyn_cast<LoadInst>(I)) + return AI->getSyncScopeID(); + if (auto *AI = dyn_cast<StoreInst>(I)) + return AI->getSyncScopeID(); + if (auto *AI = dyn_cast<FenceInst>(I)) + return AI->getSyncScopeID(); + if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) + return AI->getSyncScopeID(); + if (auto *AI = dyn_cast<AtomicRMWInst>(I)) + return AI->getSyncScopeID(); + llvm_unreachable("unhandled atomic operation"); +} + //===----------------------------------------------------------------------===// // FreezeInst Class //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 01dada25a285..06d2335821d3 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -31,7 +31,6 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" #include "llvm/Support/Casting.h" #include <cassert> @@ -39,6 +38,8 @@ namespace llvm { +class Metadata; + /// A wrapper class for inspecting calls to intrinsic functions. /// This allows the standard isa/dyncast/cast functionality to work with calls /// to intrinsic functions. @@ -472,6 +473,38 @@ public: /// @} }; +class VPCastIntrinsic : public VPIntrinsic { +public: + static bool isVPCast(Intrinsic::ID ID); + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + /// @{ + static bool classof(const IntrinsicInst *I) { + return VPCastIntrinsic::isVPCast(I->getIntrinsicID()); + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + /// @} +}; + +class VPCmpIntrinsic : public VPIntrinsic { +public: + static bool isVPCmp(Intrinsic::ID ID); + + CmpInst::Predicate getPredicate() const; + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + /// @{ + static bool classof(const IntrinsicInst *I) { + return VPCmpIntrinsic::isVPCmp(I->getIntrinsicID()); + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + /// @} +}; + /// This is the common base class for constrained floating point intrinsics. class ConstrainedFPIntrinsic : public IntrinsicInst { public: @@ -492,6 +525,9 @@ public: class ConstrainedFPCmpIntrinsic : public ConstrainedFPIntrinsic { public: FCmpInst::Predicate getPredicate() const; + bool isSignaling() const { + return getIntrinsicID() == Intrinsic::experimental_constrained_fcmps; + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -723,11 +759,6 @@ public: setArgOperand(ARG_DEST, Ptr); } - /// FIXME: Remove this function once transition to Align is over. - /// Use the version that takes MaybeAlign instead of this one. - void setDestAlignment(unsigned Alignment) { - setDestAlignment(MaybeAlign(Alignment)); - } void setDestAlignment(MaybeAlign Alignment) { removeParamAttr(ARG_DEST, Attribute::Alignment); if (Alignment) @@ -942,6 +973,7 @@ public: case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: + case Intrinsic::memset_inline: case Intrinsic::memcpy_inline: return true; default: @@ -953,12 +985,33 @@ public: } }; -/// This class wraps the llvm.memset intrinsic. +/// This class wraps the llvm.memset and llvm.memset.inline intrinsics. class MemSetInst : public MemSetBase<MemIntrinsic> { public: // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::memset; + switch (I->getIntrinsicID()) { + case Intrinsic::memset: + case Intrinsic::memset_inline: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } +}; + +/// This class wraps the llvm.memset.inline intrinsic. +class MemSetInlineInst : public MemSetInst { +public: + ConstantInt *getLength() const { + return cast<ConstantInt>(MemSetInst::getLength()); + } + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memset_inline; } static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); @@ -1043,6 +1096,7 @@ public: case Intrinsic::memcpy_inline: case Intrinsic::memmove: case Intrinsic::memset: + case Intrinsic::memset_inline: case Intrinsic::memcpy_element_unordered_atomic: case Intrinsic::memmove_element_unordered_atomic: case Intrinsic::memset_element_unordered_atomic: @@ -1064,6 +1118,7 @@ public: static bool classof(const IntrinsicInst *I) { switch (I->getIntrinsicID()) { case Intrinsic::memset: + case Intrinsic::memset_inline: case Intrinsic::memset_element_unordered_atomic: return true; default: diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 2ff48380ac28..a3db2fa59399 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -104,8 +104,8 @@ namespace Intrinsic { int lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable, StringRef Name); - /// Map a GCC builtin name to an intrinsic ID. - ID getIntrinsicForGCCBuiltin(const char *Prefix, StringRef BuiltinName); + /// Map a Clang builtin name to an intrinsic ID. + ID getIntrinsicForClangBuiltin(const char *Prefix, StringRef BuiltinName); /// Map a MS builtin name to an intrinsic ID. ID getIntrinsicForMSBuiltin(const char *Prefix, StringRef BuiltinName); @@ -142,6 +142,7 @@ namespace Intrinsic { VecOfBitcastsToInt, AMX, PPCQuad, + AnyPtrToElt, } Kind; union { @@ -180,14 +181,15 @@ namespace Intrinsic { return (ArgKind)(Argument_Info & 7); } - // VecOfAnyPtrsToElt uses both an overloaded argument (for address space) - // and a reference argument (for matching vector width and element types) + // VecOfAnyPtrsToElt and AnyPtrToElt uses both an overloaded argument (for + // address space) and a reference argument (for matching vector width and + // element types) unsigned getOverloadArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt); return Argument_Info >> 16; } unsigned getRefArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt); return Argument_Info & 0xFFFF; } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index f5248e82ad21..0dceea13ea36 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -120,6 +120,9 @@ class ReadNone<AttrIndex idx> : IntrinsicProperty { def IntrNoReturn : IntrinsicProperty; +// Applied by default. +def IntrNoCallback : IntrinsicProperty<1>; + // IntrNoSync - Threads executing the intrinsic will not synchronize using // memory or other means. Applied by default. def IntrNoSync : IntrinsicProperty<1>; @@ -212,6 +215,7 @@ class LLVMScalarOrSameVectorWidth<int idx, LLVMType elty> class LLVMPointerTo<int num> : LLVMMatchType<num>; class LLVMPointerToElt<int num> : LLVMMatchType<num>; +class LLVMAnyPointerToElt<int num> : LLVMMatchType<num>; class LLVMVectorOfAnyPointersToElt<int num> : LLVMMatchType<num>; class LLVMVectorElementType<int num> : LLVMMatchType<num>; @@ -241,6 +245,7 @@ def llvm_i8_ty : LLVMType<i8>; def llvm_i16_ty : LLVMType<i16>; def llvm_i32_ty : LLVMType<i32>; def llvm_i64_ty : LLVMType<i64>; +def llvm_i128_ty : LLVMType<i128>; def llvm_half_ty : LLVMType<f16>; def llvm_bfloat_ty : LLVMType<bf16>; def llvm_float_ty : LLVMType<f32>; @@ -380,11 +385,11 @@ class DefaultAttrsIntrinsic<list<LLVMType> ret_types, intr_properties, name, sd_properties, /*disable_default_attributes*/ 0> {} -/// GCCBuiltin - If this intrinsic exactly corresponds to a GCC builtin, this +/// ClangBuiltin - If this intrinsic exactly corresponds to a Clang builtin, this /// specifies the name of the builtin. This provides automatic CBE and CFE /// support. -class GCCBuiltin<string name> { - string GCCBuiltinName = name; +class ClangBuiltin<string name> { + string ClangBuiltinName = name; } class MSBuiltin<string name> { @@ -540,14 +545,14 @@ def int_seh_scope_end : Intrinsic<[], [], [IntrNoMem]>; // Note: we treat stacksave/stackrestore as writemem because we don't otherwise // model their dependencies on allocas. def int_stacksave : DefaultAttrsIntrinsic<[llvm_ptr_ty]>, - GCCBuiltin<"__builtin_stack_save">; + ClangBuiltin<"__builtin_stack_save">; def int_stackrestore : DefaultAttrsIntrinsic<[], [llvm_ptr_ty]>, - GCCBuiltin<"__builtin_stack_restore">; + ClangBuiltin<"__builtin_stack_restore">; def int_get_dynamic_area_offset : DefaultAttrsIntrinsic<[llvm_anyint_ty]>; def int_thread_pointer : DefaultAttrsIntrinsic<[llvm_ptr_ty], [], [IntrNoMem]>, - GCCBuiltin<"__builtin_thread_pointer">; + ClangBuiltin<"__builtin_thread_pointer">; // IntrInaccessibleMemOrArgMemOnly is a little more pessimistic than strictly // necessary for prefetch, however it does conveniently prevent the prefetch @@ -647,6 +652,17 @@ def int_memset : Intrinsic<[], NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>; +// Memset version that is guaranteed to be inlined. +// In particular this means that the generated code is not allowed to call any +// external function. +// The third argument (specifying the size) must be a constant. +def int_memset_inline + : Intrinsic<[], + [llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i1_ty], + [IntrWriteMem, IntrArgMemOnly, IntrWillReturn, IntrNoFree, + NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>, + ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>; + // FIXME: Add version of these floating point intrinsics which allow non-default // rounding modes and FP exception handling. @@ -715,7 +731,7 @@ def int_objectsize : DefaultAttrsIntrinsic<[llvm_anyint_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>, - GCCBuiltin<"__builtin_object_size">; + ClangBuiltin<"__builtin_object_size">; //===--------------- Access to Floating Point Environment -----------------===// // @@ -725,6 +741,14 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { def int_set_rounding : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; } +//===--------------- Floating Point Properties ----------------------------===// +// + +def int_is_fpclass + : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [llvm_anyfloat_ty, llvm_i32_ty], + [IntrNoMem, IntrWillReturn, ImmArg<ArgIndex<1>>]>; + //===--------------- Constrained Floating Point Intrinsics ----------------===// // @@ -909,6 +933,12 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { } // FIXME: Consider maybe adding intrinsics for sitofp, uitofp. + +// Truncate a floating point number with a specific rounding mode +def int_fptrunc_round : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], + [ llvm_anyfloat_ty, llvm_metadata_ty ], + [ IntrNoMem, IntrWillReturn ]>; + //===------------------------- Expect Intrinsics --------------------------===// // def int_expect : DefaultAttrsIntrinsic<[llvm_anyint_ty], @@ -984,12 +1014,12 @@ def int_eh_exceptioncode : Intrinsic<[llvm_i32_ty], [llvm_token_ty], [IntrNoMem] // callee-saved registers to be saved and restored (regardless of whether they // are used) in the calling function. It is used by libgcc_eh. def int_eh_unwind_init: Intrinsic<[]>, - GCCBuiltin<"__builtin_unwind_init">; + ClangBuiltin<"__builtin_unwind_init">; def int_eh_dwarf_cfa : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty]>; def int_eh_sjlj_lsda : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; -def int_eh_sjlj_callsite : Intrinsic<[], [llvm_i32_ty], [IntrNoMem]>; +def int_eh_sjlj_callsite : Intrinsic<[], [llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; def int_eh_sjlj_functioncontext : Intrinsic<[], [llvm_ptr_ty]>; def int_eh_sjlj_setjmp : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>; @@ -1025,11 +1055,11 @@ def int_init_trampoline : DefaultAttrsIntrinsic< [], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], [IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>, ReadNone<ArgIndex<1>>, ReadNone<ArgIndex<2>>]>, - GCCBuiltin<"__builtin_init_trampoline">; + ClangBuiltin<"__builtin_init_trampoline">; def int_adjust_trampoline : DefaultAttrsIntrinsic< [llvm_ptr_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>, - GCCBuiltin<"__builtin_adjust_trampoline">; + ClangBuiltin<"__builtin_adjust_trampoline">; //===------------------------ Overflow Intrinsics -------------------------===// // @@ -1309,9 +1339,9 @@ def int_coro_subfn_addr : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty], ///===-------------------------- Other Intrinsics --------------------------===// // def int_trap : Intrinsic<[], [], [IntrNoReturn, IntrCold]>, - GCCBuiltin<"__builtin_trap">; + ClangBuiltin<"__builtin_trap">; def int_debugtrap : Intrinsic<[]>, - GCCBuiltin<"__builtin_debugtrap">; + ClangBuiltin<"__builtin_debugtrap">; def int_ubsantrap : Intrinsic<[], [llvm_i8_ty], [IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; @@ -1397,14 +1427,31 @@ def int_vp_gather: DefaultAttrsIntrinsic<[ llvm_anyvector_ty], [ IntrReadMem, IntrNoSync, IntrWillReturn, IntrArgMemOnly ]>; def int_vp_scatter: DefaultAttrsIntrinsic<[], - [ llvm_anyvector_ty, - LLVMVectorOfAnyPointersToElt<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty], - [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers - -// Speculatable Binary operators -let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { + [ llvm_anyvector_ty, + LLVMVectorOfAnyPointersToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers + +// Experimental strided memory accesses +def int_experimental_vp_strided_store : DefaultAttrsIntrinsic<[], + [ llvm_anyvector_ty, + LLVMAnyPointerToElt<0>, + llvm_anyint_ty, // Stride in bytes + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture<ArgIndex<1>>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>; + +def int_experimental_vp_strided_load : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [ LLVMAnyPointerToElt<0>, + llvm_anyint_ty, // Stride in bytes + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture<ArgIndex<0>>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>; + +// Operators +let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in { + // Integer arithmetic def int_vp_add : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, @@ -1416,30 +1463,30 @@ let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] i LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty]>; def int_vp_mul : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_ashr : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_lshr : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_shl : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty]>; def int_vp_or : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_and : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, @@ -1450,35 +1497,28 @@ let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] i LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty]>; -} - -// Non-speculatable binary operators. -let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in { def int_vp_sdiv : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_udiv : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_srem : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_urem : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; -} + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; -// Floating-point arithmetic. -let IntrProperties = - [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { + // Floating-point arithmetic def int_vp_fadd : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, @@ -1490,101 +1530,169 @@ let IntrProperties = LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty]>; def int_vp_fmul : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_fdiv : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_frem : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMMatchType<0>, - LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; -} -// Shuffles. -def int_vp_select : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - LLVMMatchType<0>, - LLVMMatchType<0>, - llvm_i32_ty]>; - -def int_vp_merge : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], - [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - LLVMMatchType<0>, - LLVMMatchType<0>, - llvm_i32_ty]>; - -// Reductions -let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fneg : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fma : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + + // Casts + def int_vp_trunc : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_zext : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_sext : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fptrunc : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fpext : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fptoui : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_fptosi : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_uitofp : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_sitofp : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_ptrtoint : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_inttoptr : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + + // Shuffles + def int_vp_select : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty]>; + def int_vp_merge : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty]>; + + // Comparisons + def int_vp_fcmp : DefaultAttrsIntrinsic<[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ], + [ llvm_anyvector_ty, + LLVMMatchType<0>, + llvm_metadata_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_icmp : DefaultAttrsIntrinsic<[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ], + [ llvm_anyvector_ty, + LLVMMatchType<0>, + llvm_metadata_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + + // Reductions def int_vp_reduce_fadd : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_fmul : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_add : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_mul : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_and : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_or : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_xor : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_smax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_smin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_umax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_umin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_fmax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; def int_vp_reduce_fmin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], - [LLVMVectorElementType<0>, - llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_i32_ty]>; + [ LLVMVectorElementType<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; } def int_get_active_lane_mask: @@ -1840,28 +1948,26 @@ def int_preserve_struct_access_index : DefaultAttrsIntrinsic<[llvm_anyptr_ty], //===------------ Intrinsics to perform common vector shuffles ------------===// def int_experimental_vector_reverse : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>], - [IntrNoMem]>; + [LLVMMatchType<0>], + [IntrNoMem]>; -//===---------- Intrinsics to query properties of scalable vectors --------===// -def int_vscale : DefaultAttrsIntrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; - -//===---------- Intrinsics to perform subvector insertion/extraction ------===// -def int_experimental_vector_insert : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty], - [IntrNoMem, ImmArg<ArgIndex<2>>]>; - -def int_experimental_vector_extract : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_i64_ty], - [IntrNoMem, ImmArg<ArgIndex<1>>]>; - -//===---------- Named shufflevector intrinsics ------===// def int_experimental_vector_splice : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; +//===---------- Intrinsics to query properties of scalable vectors --------===// +def int_vscale : DefaultAttrsIntrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; + +//===---------- Intrinsics to perform subvector insertion/extraction ------===// +def int_vector_insert : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<ArgIndex<2>>]>; + +def int_vector_extract : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<ArgIndex<1>>]>; //===----------------- Pointer Authentication Intrinsics ------------------===// // @@ -1936,4 +2042,6 @@ include "llvm/IR/IntrinsicsBPF.td" include "llvm/IR/IntrinsicsSystemZ.td" include "llvm/IR/IntrinsicsWebAssembly.td" include "llvm/IR/IntrinsicsRISCV.td" +include "llvm/IR/IntrinsicsSPIRV.td" include "llvm/IR/IntrinsicsVE.td" +include "llvm/IR/IntrinsicsDirectX.td" diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index a65ddff07a29..1256ab2c9f84 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -62,14 +62,17 @@ def int_aarch64_frint64x def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; +def int_aarch64_break : Intrinsic<[], [llvm_i32_ty], + [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; + //===----------------------------------------------------------------------===// // Data Barrier Instructions -def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, +def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; -def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, +def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; -def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, +def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; // A space-consuming intrinsic primarily for testing block and jump table @@ -907,15 +910,15 @@ let TargetPrefix = "aarch64" in { // Transactional Memory Extension (TME) Intrinsics let TargetPrefix = "aarch64" in { -def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">, +def int_aarch64_tstart : ClangBuiltin<"__builtin_arm_tstart">, Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>; -def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>; +def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>; -def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">, +def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">, Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>; -def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">, +def int_aarch64_ttest : ClangBuiltin<"__builtin_arm_ttest">, Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; @@ -1759,10 +1762,10 @@ def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; // FFR manipulation // -def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; -def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; -def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; -def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_rdffr : ClangBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_rdffr_z : ClangBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_setffr : ClangBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_wrffr : ClangBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; // // Saturating scalar arithmetic @@ -2493,31 +2496,31 @@ def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; // SVE2 - Optional AES, SHA-3 and SM4 // -def int_aarch64_sve_aesd : GCCBuiltin<"__builtin_sve_svaesd_u8">, +def int_aarch64_sve_aesd : ClangBuiltin<"__builtin_sve_svaesd_u8">, DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], [llvm_nxv16i8_ty, llvm_nxv16i8_ty], [IntrNoMem]>; -def int_aarch64_sve_aesimc : GCCBuiltin<"__builtin_sve_svaesimc_u8">, +def int_aarch64_sve_aesimc : ClangBuiltin<"__builtin_sve_svaesimc_u8">, DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], [llvm_nxv16i8_ty], [IntrNoMem]>; -def int_aarch64_sve_aese : GCCBuiltin<"__builtin_sve_svaese_u8">, +def int_aarch64_sve_aese : ClangBuiltin<"__builtin_sve_svaese_u8">, DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], [llvm_nxv16i8_ty, llvm_nxv16i8_ty], [IntrNoMem]>; -def int_aarch64_sve_aesmc : GCCBuiltin<"__builtin_sve_svaesmc_u8">, +def int_aarch64_sve_aesmc : ClangBuiltin<"__builtin_sve_svaesmc_u8">, DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], [llvm_nxv16i8_ty], [IntrNoMem]>; -def int_aarch64_sve_rax1 : GCCBuiltin<"__builtin_sve_svrax1_u64">, +def int_aarch64_sve_rax1 : ClangBuiltin<"__builtin_sve_svrax1_u64">, DefaultAttrsIntrinsic<[llvm_nxv2i64_ty], [llvm_nxv2i64_ty, llvm_nxv2i64_ty], [IntrNoMem]>; -def int_aarch64_sve_sm4e : GCCBuiltin<"__builtin_sve_svsm4e_u32">, +def int_aarch64_sve_sm4e : ClangBuiltin<"__builtin_sve_svsm4e_u32">, DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], [llvm_nxv4i32_ty, llvm_nxv4i32_ty], [IntrNoMem]>; -def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">, +def int_aarch64_sve_sm4ekey : ClangBuiltin<"__builtin_sve_svsm4ekey_u32">, DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], [llvm_nxv4i32_ty, llvm_nxv4i32_ty], [IntrNoMem]>; @@ -2580,3 +2583,130 @@ def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; + +// Scalable Matrix Extension (SME) Intrinsics +let TargetPrefix = "aarch64" in { + class SME_Load_Store_Intrinsic<LLVMType pred_ty> + : DefaultAttrsIntrinsic<[], + [pred_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>; + + // Loads + def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + + // Stores + def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; + + // Spill + fill + def int_aarch64_sme_ldr : DefaultAttrsIntrinsic< + [], [llvm_i32_ty, llvm_ptr_ty]>; + def int_aarch64_sme_str : DefaultAttrsIntrinsic< + [], [llvm_i32_ty, llvm_ptr_ty]>; + + class SME_TileToVector_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i64_ty, llvm_i32_ty]>; + class SME_VectorToTile_Intrinsic + : DefaultAttrsIntrinsic<[], + [llvm_i64_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty]>; + + def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic; + def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic; + def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic; + def int_aarch64_sme_write_vert : SME_VectorToTile_Intrinsic; + + def int_aarch64_sme_readq_horiz : SME_TileToVector_Intrinsic; + def int_aarch64_sme_readq_vert : SME_TileToVector_Intrinsic; + def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic; + def int_aarch64_sme_writeq_vert : SME_VectorToTile_Intrinsic; + + def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i64_ty]>; + + class SME_OuterProduct_Intrinsic + : DefaultAttrsIntrinsic<[], + [llvm_i64_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + llvm_anyvector_ty]>; + + def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic; + + def int_aarch64_sme_mopa_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_mops_wide : SME_OuterProduct_Intrinsic; + + def int_aarch64_sme_smopa_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_smops_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_umopa_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_umops_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_sumopa_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_sumops_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_usmopa_wide : SME_OuterProduct_Intrinsic; + def int_aarch64_sme_usmops_wide : SME_OuterProduct_Intrinsic; + + // + // Counting elements + // + + class AdvSIMD_SME_CNTSB_Intrinsic + : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>; + + def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic; + def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic; + def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic; + def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic; + + // + // PSTATE Functions + // + + def int_aarch64_sme_get_pstatesm + : DefaultAttrsIntrinsic<[llvm_i64_ty], [], + [IntrReadMem, IntrInaccessibleMemOnly]>; + + def int_aarch64_sme_get_tpidr2 + : DefaultAttrsIntrinsic<[llvm_i64_ty], [], + [IntrNoMem, IntrHasSideEffects]>; + def int_aarch64_sme_set_tpidr2 + : DefaultAttrsIntrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + // Clamp + // + + def int_aarch64_sve_sclamp : AdvSIMD_3VectorArg_Intrinsic; + def int_aarch64_sve_uclamp : AdvSIMD_3VectorArg_Intrinsic; + + // + // Reversal + // + + def int_aarch64_sve_revd : AdvSIMD_Merged1VectorArg_Intrinsic; + + // + // Predicate selection + // + + def int_aarch64_sve_psel + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, llvm_i32_ty]>; +} diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index c5d266eb57ec..c2dcfc254568 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -14,7 +14,7 @@ class AMDGPUReadPreloadRegisterIntrinsic : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; class AMDGPUReadPreloadRegisterIntrinsicNamed<string name> - : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>, GCCBuiltin<name>; + : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>, ClangBuiltin<name>; // Used to tag image and resource intrinsics with information used to generate // mem operands. @@ -47,12 +47,12 @@ defm int_r600_read_tgid : AMDGPUReadPreloadRegisterIntrinsic_xyz_named defm int_r600_read_local_size : AMDGPUReadPreloadRegisterIntrinsic_xyz; defm int_r600_read_tidig : AMDGPUReadPreloadRegisterIntrinsic_xyz; -def int_r600_group_barrier : GCCBuiltin<"__builtin_r600_group_barrier">, +def int_r600_group_barrier : ClangBuiltin<"__builtin_r600_group_barrier">, Intrinsic<[], [], [IntrConvergent, IntrWillReturn]>; // AS 7 is PARAM_I_ADDRESS, used for kernel arguments def int_r600_implicitarg_ptr : - GCCBuiltin<"__builtin_r600_implicitarg_ptr">, + ClangBuiltin<"__builtin_r600_implicitarg_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 7>], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; @@ -61,7 +61,7 @@ def int_r600_rat_store_typed : // 2nd parameter: Index // 3rd parameter: Constant RAT ID Intrinsic<[], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrWillReturn]>, - GCCBuiltin<"__builtin_r600_rat_store_typed">; + ClangBuiltin<"__builtin_r600_rat_store_typed">; def int_r600_recipsqrt_ieee : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] @@ -145,30 +145,30 @@ def int_amdgcn_dispatch_ptr : [Align<RetIndex, 4>, IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_queue_ptr : - GCCBuiltin<"__builtin_amdgcn_queue_ptr">, + ClangBuiltin<"__builtin_amdgcn_queue_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 4>], [], [Align<RetIndex, 4>, IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_kernarg_segment_ptr : - GCCBuiltin<"__builtin_amdgcn_kernarg_segment_ptr">, + ClangBuiltin<"__builtin_amdgcn_kernarg_segment_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 4>], [], [Align<RetIndex, 4>, IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_implicitarg_ptr : - GCCBuiltin<"__builtin_amdgcn_implicitarg_ptr">, + ClangBuiltin<"__builtin_amdgcn_implicitarg_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 4>], [], [Align<RetIndex, 4>, IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_groupstaticsize : - GCCBuiltin<"__builtin_amdgcn_groupstaticsize">, + ClangBuiltin<"__builtin_amdgcn_groupstaticsize">, Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_dispatch_id : - GCCBuiltin<"__builtin_amdgcn_dispatch_id">, + ClangBuiltin<"__builtin_amdgcn_dispatch_id">, Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; def int_amdgcn_implicit_buffer_ptr : - GCCBuiltin<"__builtin_amdgcn_implicit_buffer_ptr">, + ClangBuiltin<"__builtin_amdgcn_implicit_buffer_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 4>], [], [Align<RetIndex, 4>, IntrNoMem, IntrSpeculatable, IntrWillReturn]>; @@ -190,7 +190,7 @@ def int_amdgcn_init_exec_from_input : Intrinsic<[], [IntrConvergent, ImmArg<ArgIndex<1>>]>; def int_amdgcn_wavefrontsize : - GCCBuiltin<"__builtin_amdgcn_wavefrontsize">, + ClangBuiltin<"__builtin_amdgcn_wavefrontsize">, Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; @@ -200,20 +200,44 @@ def int_amdgcn_wavefrontsize : // The first parameter is s_sendmsg immediate (i16), // the second one is copied to m0 -def int_amdgcn_s_sendmsg : GCCBuiltin<"__builtin_amdgcn_s_sendmsg">, +def int_amdgcn_s_sendmsg : ClangBuiltin<"__builtin_amdgcn_s_sendmsg">, Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>; -def int_amdgcn_s_sendmsghalt : GCCBuiltin<"__builtin_amdgcn_s_sendmsghalt">, +def int_amdgcn_s_sendmsghalt : ClangBuiltin<"__builtin_amdgcn_s_sendmsghalt">, Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>; -def int_amdgcn_s_barrier : GCCBuiltin<"__builtin_amdgcn_s_barrier">, + +// gfx11 intrinsic +// The first parameter is s_sendmsg immediate (i16). Return type is i32 or i64. +def int_amdgcn_s_sendmsg_rtn : Intrinsic <[llvm_anyint_ty], [llvm_i32_ty], + [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>; + +def int_amdgcn_s_barrier : ClangBuiltin<"__builtin_amdgcn_s_barrier">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn]>; -def int_amdgcn_wave_barrier : GCCBuiltin<"__builtin_amdgcn_wave_barrier">, +def int_amdgcn_wave_barrier : ClangBuiltin<"__builtin_amdgcn_wave_barrier">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn]>; -def int_amdgcn_s_waitcnt : GCCBuiltin<"__builtin_amdgcn_s_waitcnt">, +// The 1st parameter is a mask for the types of instructions that may be allowed +// to cross the SCHED_BARRIER during scheduling. +// MASK = 0x0000 0000: No instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0001: ALL, non-memory, non-side-effect producing instructions may be +// scheduled across SCHED_BARRIER, i.e. allow ALU instructions to pass. +// MASK = 0x0000 0002: VALU instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0004: SALU instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0008: MFMA instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0010: ALL VMEM instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0020: VMEM read instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0040: VMEM write instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0080: ALL DS instructions may be scheduled across SCHED_BARRIER. +// MASK = 0x0000 0100: ALL DS read instructions may be scheduled accoss SCHED_BARRIER. +// MASK = 0x0000 0200: ALL DS write instructions may be scheduled across SCHED_BARRIER. +def int_amdgcn_sched_barrier : ClangBuiltin<"__builtin_amdgcn_sched_barrier">, + Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent, + IntrWillReturn]>; + +def int_amdgcn_s_waitcnt : ClangBuiltin<"__builtin_amdgcn_s_waitcnt">, Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_div_scale : Intrinsic< @@ -255,7 +279,7 @@ def int_amdgcn_log_clamp : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_fmul_legacy : GCCBuiltin<"__builtin_amdgcn_fmul_legacy">, +def int_amdgcn_fmul_legacy : ClangBuiltin<"__builtin_amdgcn_fmul_legacy">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; @@ -274,7 +298,7 @@ def int_amdgcn_rcp : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_rcp_legacy : GCCBuiltin<"__builtin_amdgcn_rcp_legacy">, +def int_amdgcn_rcp_legacy : ClangBuiltin<"__builtin_amdgcn_rcp_legacy">, Intrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; @@ -287,7 +311,7 @@ def int_amdgcn_rsq : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_rsq_legacy : GCCBuiltin<"__builtin_amdgcn_rsq_legacy">, +def int_amdgcn_rsq_legacy : ClangBuiltin<"__builtin_amdgcn_rsq_legacy">, Intrinsic< [llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; @@ -316,31 +340,31 @@ def int_amdgcn_fract : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cvt_pkrtz : GCCBuiltin<"__builtin_amdgcn_cvt_pkrtz">, +def int_amdgcn_cvt_pkrtz : ClangBuiltin<"__builtin_amdgcn_cvt_pkrtz">, Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_cvt_pknorm_i16 : - GCCBuiltin<"__builtin_amdgcn_cvt_pknorm_i16">, + ClangBuiltin<"__builtin_amdgcn_cvt_pknorm_i16">, Intrinsic<[llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_cvt_pknorm_u16 : - GCCBuiltin<"__builtin_amdgcn_cvt_pknorm_u16">, + ClangBuiltin<"__builtin_amdgcn_cvt_pknorm_u16">, Intrinsic<[llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_cvt_pk_i16 : - GCCBuiltin<"__builtin_amdgcn_cvt_pk_i16">, + ClangBuiltin<"__builtin_amdgcn_cvt_pk_i16">, Intrinsic< [llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cvt_pk_u16 : GCCBuiltin<"__builtin_amdgcn_cvt_pk_u16">, +def int_amdgcn_cvt_pk_u16 : ClangBuiltin<"__builtin_amdgcn_cvt_pk_u16">, Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; @@ -350,31 +374,31 @@ def int_amdgcn_class : Intrinsic< [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_fmed3 : GCCBuiltin<"__builtin_amdgcn_fmed3">, +def int_amdgcn_fmed3 : ClangBuiltin<"__builtin_amdgcn_fmed3">, Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cubeid : GCCBuiltin<"__builtin_amdgcn_cubeid">, +def int_amdgcn_cubeid : ClangBuiltin<"__builtin_amdgcn_cubeid">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cubema : GCCBuiltin<"__builtin_amdgcn_cubema">, +def int_amdgcn_cubema : ClangBuiltin<"__builtin_amdgcn_cubema">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cubesc : GCCBuiltin<"__builtin_amdgcn_cubesc">, +def int_amdgcn_cubesc : ClangBuiltin<"__builtin_amdgcn_cubesc">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; -def int_amdgcn_cubetc : GCCBuiltin<"__builtin_amdgcn_cubetc">, +def int_amdgcn_cubetc : ClangBuiltin<"__builtin_amdgcn_cubetc">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] @@ -838,6 +862,13 @@ defset list<AMDGPUImageDimIntrinsic> AMDGPUImageDimIntrinsics = { [IntrReadMem], [SDNPMemOperand]>; } + foreach dim = AMDGPUDims.Msaa in { + def int_amdgcn_image_msaa_load # _ # dim.Name: + AMDGPUImageDimIntrinsic< + AMDGPUDimNoSampleProfile<"MSAA_LOAD", dim, [llvm_any_ty], []>, + [IntrReadMem], [SDNPMemOperand]>; + } + ////////////////////////////////////////////////////////////////////////// // sample and getlod intrinsics ////////////////////////////////////////////////////////////////////////// @@ -949,10 +980,12 @@ class AMDGPUBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic < def int_amdgcn_buffer_load_format : AMDGPUBufferLoad<llvm_anyfloat_ty>; def int_amdgcn_buffer_load : AMDGPUBufferLoad; +// Generate a buffer_load instruction that may be optimized to s_buffer_load if +// the offset argument is uniform. def int_amdgcn_s_buffer_load : Intrinsic < [llvm_any_ty], [llvm_v4i32_ty, // rsrc(SGPR) - llvm_i32_ty, // byte offset(SGPR/imm) + llvm_i32_ty, // byte offset llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 2 = dlc) [IntrNoMem, IntrWillReturn, ImmArg<ArgIndex<2>>]>, AMDGPURsrcIntrinsic<0>; @@ -1259,6 +1292,40 @@ class AMDGPUBufferAtomicFP : Intrinsic < // Legacy form of the intrinsic. raw and struct forms should be preferred. def int_amdgcn_buffer_atomic_fadd : AMDGPUBufferAtomicFP; + +class AMDGPURawBufferLoadLDS : Intrinsic < + [], + [llvm_v4i32_ty, // rsrc(SGPR) + LLVMQualPointerType<llvm_i8_ty, 3>, // LDS base offset + llvm_i32_ty, // Data byte size: 1/2/4 + llvm_i32_ty, // voffset(VGPR, included in bounds checking and swizzling) + llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) + llvm_i32_ty, // imm offset(imm, included in bounds checking and swizzling) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+)) + // swizzled buffer (bit 3 = swz)) + [IntrWillReturn, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>, + ImmArg<ArgIndex<6>>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; +def int_amdgcn_raw_buffer_load_lds : AMDGPURawBufferLoadLDS; + +class AMDGPUStructBufferLoadLDS : Intrinsic < + [], + [llvm_v4i32_ty, // rsrc(SGPR) + LLVMQualPointerType<llvm_i8_ty, 3>, // LDS base offset + llvm_i32_ty, // Data byte size: 1/2/4 + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // voffset(VGPR, included in bounds checking and swizzling) + llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) + llvm_i32_ty, // imm offset(imm, included in bounds checking and swizzling) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc, + // bit 1 = slc, + // bit 2 = dlc on gfx10+)) + // swizzled buffer (bit 3 = swz)) + [IntrWillReturn, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<6>>, + ImmArg<ArgIndex<7>>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; +def int_amdgcn_struct_buffer_load_lds : AMDGPUStructBufferLoadLDS; + } // defset AMDGPUBufferIntrinsics // Uses that do not set the done bit should set IntrWriteMem on the @@ -1278,7 +1345,21 @@ def int_amdgcn_exp : Intrinsic <[], [ IntrWillReturn] >; -// exp with compr bit set. +// exp with row_en bit set. Only supported on GFX11+. +def int_amdgcn_exp_row : Intrinsic <[], [ + llvm_i32_ty, // tgt, + llvm_i32_ty, // en + llvm_any_ty, // src0 (f32 or i32) + LLVMMatchType<0>, // src1 + LLVMMatchType<0>, // src2 + LLVMMatchType<0>, // src3 + llvm_i1_ty, // done + llvm_i32_ty], // row number + [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<6>>, + IntrWriteMem, IntrInaccessibleMemOnly, IntrWillReturn] +>; + +// exp with compr bit set. Not supported on GFX11+. def int_amdgcn_exp_compr : Intrinsic <[], [ llvm_i32_ty, // tgt, llvm_i32_ty, // en @@ -1292,35 +1373,35 @@ def int_amdgcn_exp_compr : Intrinsic <[], [ >; def int_amdgcn_buffer_wbinvl1_sc : - GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_sc">, + ClangBuiltin<"__builtin_amdgcn_buffer_wbinvl1_sc">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_buffer_wbinvl1 : - GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1">, + ClangBuiltin<"__builtin_amdgcn_buffer_wbinvl1">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_s_dcache_inv : - GCCBuiltin<"__builtin_amdgcn_s_dcache_inv">, + ClangBuiltin<"__builtin_amdgcn_s_dcache_inv">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_s_memtime : - GCCBuiltin<"__builtin_amdgcn_s_memtime">, + ClangBuiltin<"__builtin_amdgcn_s_memtime">, Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_s_sleep : - GCCBuiltin<"__builtin_amdgcn_s_sleep">, + ClangBuiltin<"__builtin_amdgcn_s_sleep">, Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]> { } def int_amdgcn_s_incperflevel : - GCCBuiltin<"__builtin_amdgcn_s_incperflevel">, + ClangBuiltin<"__builtin_amdgcn_s_incperflevel">, Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]> { } def int_amdgcn_s_decperflevel : - GCCBuiltin<"__builtin_amdgcn_s_decperflevel">, + ClangBuiltin<"__builtin_amdgcn_s_decperflevel">, Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]> { } @@ -1329,11 +1410,16 @@ def int_amdgcn_s_sethalt : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; +def int_amdgcn_s_setprio : + ClangBuiltin<"__builtin_amdgcn_s_setprio">, + Intrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, + IntrHasSideEffects, IntrWillReturn]>; + +// This is IntrHasSideEffects so it can be used to read cycle counters. def int_amdgcn_s_getreg : - GCCBuiltin<"__builtin_amdgcn_s_getreg">, + ClangBuiltin<"__builtin_amdgcn_s_getreg">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrReadMem, IntrSpeculatable, - IntrWillReturn, ImmArg<ArgIndex<0>>] + [IntrNoMem, IntrHasSideEffects, IntrWillReturn, ImmArg<ArgIndex<0>>] >; // Note this can be used to set FP environment properties that are @@ -1341,7 +1427,7 @@ def int_amdgcn_s_getreg : // available (and value required to access them) may differ per // subtarget. llvm.amdgcn.s.setreg(hwmode, value) def int_amdgcn_s_setreg : - GCCBuiltin<"__builtin_amdgcn_s_setreg">, + ClangBuiltin<"__builtin_amdgcn_s_setreg">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrWillReturn, ImmArg<ArgIndex<0>>] >; @@ -1353,14 +1439,14 @@ def int_amdgcn_s_setreg : // produce the desired results as optimizations may cause code movement, // especially as we explicitly use IntrNoMem to allow optimizations. def int_amdgcn_s_getpc : - GCCBuiltin<"__builtin_amdgcn_s_getpc">, + ClangBuiltin<"__builtin_amdgcn_s_getpc">, Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; // __builtin_amdgcn_interp_mov <param>, <attr_chan>, <attr>, <m0> // param values: 0 = P10, 1 = P20, 2 = P0 def int_amdgcn_interp_mov : - GCCBuiltin<"__builtin_amdgcn_interp_mov">, + ClangBuiltin<"__builtin_amdgcn_interp_mov">, Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, @@ -1370,7 +1456,7 @@ def int_amdgcn_interp_mov : // This intrinsic reads from lds, but the memory values are constant, // so it behaves like IntrNoMem. def int_amdgcn_interp_p1 : - GCCBuiltin<"__builtin_amdgcn_interp_p1">, + ClangBuiltin<"__builtin_amdgcn_interp_p1">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, @@ -1378,7 +1464,7 @@ def int_amdgcn_interp_p1 : // __builtin_amdgcn_interp_p2 <p1>, <j>, <attr_chan>, <attr>, <m0> def int_amdgcn_interp_p2 : - GCCBuiltin<"__builtin_amdgcn_interp_p2">, + ClangBuiltin<"__builtin_amdgcn_interp_p2">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, @@ -1388,7 +1474,7 @@ def int_amdgcn_interp_p2 : // __builtin_amdgcn_interp_p1_f16 <i>, <attr_chan>, <attr>, <high>, <m0> // high selects whether high or low 16-bits are loaded from LDS def int_amdgcn_interp_p1_f16 : - GCCBuiltin<"__builtin_amdgcn_interp_p1_f16">, + ClangBuiltin<"__builtin_amdgcn_interp_p1_f16">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, @@ -1397,12 +1483,57 @@ def int_amdgcn_interp_p1_f16 : // __builtin_amdgcn_interp_p2_f16 <p1>, <j>, <attr_chan>, <attr>, <high>, <m0> // high selects whether high or low 16-bits are loaded from LDS def int_amdgcn_interp_p2_f16 : - GCCBuiltin<"__builtin_amdgcn_interp_p2_f16">, + ClangBuiltin<"__builtin_amdgcn_interp_p2_f16">, Intrinsic<[llvm_half_ty], [llvm_float_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; +// llvm.amdgcn.lds.direct.load <m0> +// The input argument is m0, which contains a packed combination of address +// offset and flags describing the data type. +def int_amdgcn_lds_direct_load : + Intrinsic<[llvm_any_ty], // overloaded for types u8, u16, i32/f32, i8, i16 + [llvm_i32_ty], + [IntrReadMem, IntrSpeculatable, IntrWillReturn]>; + +// llvm.amdgcn.lds.param.load <attr_chan>, <attr>, <m0> +// Like interp intrinsics, this reads from lds, but the memory values are constant, +// so it behaves like IntrNoMem. +def int_amdgcn_lds_param_load : + Intrinsic<[llvm_float_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; + +// llvm.amdgcn.interp.inreg.p10 <p>, <i>, <p0> +def int_amdgcn_interp_inreg_p10 : + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; + +// llvm.amdgcn.interp.inreg.p2 <p>, <j>, <tmp> +def int_amdgcn_interp_inreg_p2 : + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; + +// llvm.amdgcn.interp.inreg.p10.f16 <p>, <i>, <p0>, <high> +// high selects whether high or low 16-bits are used for p and p0 operands +def int_amdgcn_interp_inreg_p10_f16: + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i1_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<3>>]>; + +// llvm.amdgcn.interp.inreg.p2.f16 <p>, <j>, <tmp>, <high> +// high selects whether high or low 16-bits are used for p operand +def int_amdgcn_interp_inreg_p2_f16 : + Intrinsic<[llvm_half_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i1_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<3>>]>; + // Deprecated: use llvm.amdgcn.live.mask instead. def int_amdgcn_ps_live : Intrinsic < [llvm_i1_ty], @@ -1416,18 +1547,18 @@ def int_amdgcn_live_mask : Intrinsic <[llvm_i1_ty], >; def int_amdgcn_mbcnt_lo : - GCCBuiltin<"__builtin_amdgcn_mbcnt_lo">, + ClangBuiltin<"__builtin_amdgcn_mbcnt_lo">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; def int_amdgcn_mbcnt_hi : - GCCBuiltin<"__builtin_amdgcn_mbcnt_hi">, + ClangBuiltin<"__builtin_amdgcn_mbcnt_hi">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; // llvm.amdgcn.ds.swizzle src offset def int_amdgcn_ds_swizzle : - GCCBuiltin<"__builtin_amdgcn_ds_swizzle">, + ClangBuiltin<"__builtin_amdgcn_ds_swizzle">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg<ArgIndex<1>>]>; @@ -1443,55 +1574,55 @@ def int_amdgcn_sbfe : Intrinsic<[llvm_anyint_ty], >; def int_amdgcn_lerp : - GCCBuiltin<"__builtin_amdgcn_lerp">, + ClangBuiltin<"__builtin_amdgcn_lerp">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_sad_u8 : - GCCBuiltin<"__builtin_amdgcn_sad_u8">, + ClangBuiltin<"__builtin_amdgcn_sad_u8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_msad_u8 : - GCCBuiltin<"__builtin_amdgcn_msad_u8">, + ClangBuiltin<"__builtin_amdgcn_msad_u8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_sad_hi_u8 : - GCCBuiltin<"__builtin_amdgcn_sad_hi_u8">, + ClangBuiltin<"__builtin_amdgcn_sad_hi_u8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_sad_u16 : - GCCBuiltin<"__builtin_amdgcn_sad_u16">, + ClangBuiltin<"__builtin_amdgcn_sad_u16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_qsad_pk_u16_u8 : - GCCBuiltin<"__builtin_amdgcn_qsad_pk_u16_u8">, + ClangBuiltin<"__builtin_amdgcn_qsad_pk_u16_u8">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_mqsad_pk_u16_u8 : - GCCBuiltin<"__builtin_amdgcn_mqsad_pk_u16_u8">, + ClangBuiltin<"__builtin_amdgcn_mqsad_pk_u16_u8">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_mqsad_u32_u8 : - GCCBuiltin<"__builtin_amdgcn_mqsad_u32_u8">, + ClangBuiltin<"__builtin_amdgcn_mqsad_u32_u8">, Intrinsic<[llvm_v4i32_ty], [llvm_i64_ty, llvm_i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; def int_amdgcn_cvt_pk_u8_f32 : - GCCBuiltin<"__builtin_amdgcn_cvt_pk_u8_f32">, + ClangBuiltin<"__builtin_amdgcn_cvt_pk_u8_f32">, Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; @@ -1511,14 +1642,14 @@ def int_amdgcn_ballot : [IntrNoMem, IntrConvergent, IntrWillReturn]>; def int_amdgcn_readfirstlane : - GCCBuiltin<"__builtin_amdgcn_readfirstlane">, + ClangBuiltin<"__builtin_amdgcn_readfirstlane">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn]>; // The lane argument must be uniform across the currently active threads of the // current wave. Otherwise, the result is undefined. def int_amdgcn_readlane : - GCCBuiltin<"__builtin_amdgcn_readlane">, + ClangBuiltin<"__builtin_amdgcn_readlane">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn]>; @@ -1526,7 +1657,7 @@ def int_amdgcn_readlane : // currently active threads of the current wave. Otherwise, the result is // undefined. def int_amdgcn_writelane : - GCCBuiltin<"__builtin_amdgcn_writelane">, + ClangBuiltin<"__builtin_amdgcn_writelane">, Intrinsic<[llvm_i32_ty], [ llvm_i32_ty, // uniform value to write: returned by the selected lane llvm_i32_ty, // uniform lane select @@ -1535,7 +1666,7 @@ def int_amdgcn_writelane : [IntrNoMem, IntrConvergent, IntrWillReturn] >; -def int_amdgcn_alignbyte : GCCBuiltin<"__builtin_amdgcn_alignbyte">, +def int_amdgcn_alignbyte : ClangBuiltin<"__builtin_amdgcn_alignbyte">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] >; @@ -1565,7 +1696,7 @@ def int_amdgcn_mulhi_u24 : Intrinsic<[llvm_i32_ty], // bar_val is the total number of waves that will wait on this // barrier, minus 1. def int_amdgcn_ds_gws_init : - GCCBuiltin<"__builtin_amdgcn_ds_gws_init">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_init">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrWriteMem, @@ -1577,7 +1708,7 @@ def int_amdgcn_ds_gws_init : // bar_val is the total number of waves that will wait on this // barrier, minus 1. def int_amdgcn_ds_gws_barrier : - GCCBuiltin<"__builtin_amdgcn_ds_gws_barrier">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_barrier">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrInaccessibleMemOnly, IntrWillReturn], "", @@ -1586,7 +1717,7 @@ def int_amdgcn_ds_gws_barrier : // llvm.amdgcn.ds.gws.sema.v(i32 resource_id) def int_amdgcn_ds_gws_sema_v : - GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_v">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_sema_v">, Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrInaccessibleMemOnly, IntrWillReturn], "", @@ -1595,7 +1726,7 @@ def int_amdgcn_ds_gws_sema_v : // llvm.amdgcn.ds.gws.sema.br(i32 vsrc, i32 resource_id) def int_amdgcn_ds_gws_sema_br : - GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_br">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_sema_br">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrInaccessibleMemOnly, IntrWillReturn], "", @@ -1604,7 +1735,7 @@ def int_amdgcn_ds_gws_sema_br : // llvm.amdgcn.ds.gws.sema.p(i32 resource_id) def int_amdgcn_ds_gws_sema_p : - GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_p">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_sema_p">, Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrInaccessibleMemOnly, IntrWillReturn], "", @@ -1613,7 +1744,7 @@ def int_amdgcn_ds_gws_sema_p : // llvm.amdgcn.ds.gws.sema.release.all(i32 resource_id) def int_amdgcn_ds_gws_sema_release_all : - GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_release_all">, + ClangBuiltin<"__builtin_amdgcn_ds_gws_sema_release_all">, Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrInaccessibleMemOnly, IntrWillReturn], "", @@ -1644,7 +1775,7 @@ def int_amdgcn_wqm_vote : Intrinsic<[llvm_i1_ty], // FIXME: Should this be IntrNoMem, IntrHasSideEffects, or IntrWillReturn? def int_amdgcn_kill : Intrinsic<[], [llvm_i1_ty], []>; -def int_amdgcn_endpgm : GCCBuiltin<"__builtin_amdgcn_endpgm">, +def int_amdgcn_endpgm : ClangBuiltin<"__builtin_amdgcn_endpgm">, Intrinsic<[], [], [IntrNoReturn, IntrCold, IntrNoMem, IntrHasSideEffects] >; @@ -1683,13 +1814,13 @@ def int_amdgcn_set_inactive : [IntrNoMem, IntrConvergent, IntrWillReturn]>; // Return if the given flat pointer points to a local memory address. -def int_amdgcn_is_shared : GCCBuiltin<"__builtin_amdgcn_is_shared">, +def int_amdgcn_is_shared : ClangBuiltin<"__builtin_amdgcn_is_shared">, Intrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>, IntrWillReturn] >; // Return if the given flat pointer points to a prvate memory address. -def int_amdgcn_is_private : GCCBuiltin<"__builtin_amdgcn_is_private">, +def int_amdgcn_is_private : ClangBuiltin<"__builtin_amdgcn_is_private">, Intrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>, IntrWillReturn] >; @@ -1699,11 +1830,11 @@ def int_amdgcn_is_private : GCCBuiltin<"__builtin_amdgcn_is_private">, //===----------------------------------------------------------------------===// def int_amdgcn_s_dcache_inv_vol : - GCCBuiltin<"__builtin_amdgcn_s_dcache_inv_vol">, + ClangBuiltin<"__builtin_amdgcn_s_dcache_inv_vol">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_buffer_wbinvl1_vol : - GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_vol">, + ClangBuiltin<"__builtin_amdgcn_buffer_wbinvl1_vol">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; //===----------------------------------------------------------------------===// @@ -1732,48 +1863,67 @@ def int_amdgcn_update_dpp : ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; def int_amdgcn_s_dcache_wb : - GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, + ClangBuiltin<"__builtin_amdgcn_s_dcache_wb">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_s_dcache_wb_vol : - GCCBuiltin<"__builtin_amdgcn_s_dcache_wb_vol">, + ClangBuiltin<"__builtin_amdgcn_s_dcache_wb_vol">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; def int_amdgcn_s_memrealtime : - GCCBuiltin<"__builtin_amdgcn_s_memrealtime">, + ClangBuiltin<"__builtin_amdgcn_s_memrealtime">, Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; // llvm.amdgcn.ds.permute <index> <src> def int_amdgcn_ds_permute : - GCCBuiltin<"__builtin_amdgcn_ds_permute">, + ClangBuiltin<"__builtin_amdgcn_ds_permute">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn]>; // llvm.amdgcn.ds.bpermute <index> <src> def int_amdgcn_ds_bpermute : - GCCBuiltin<"__builtin_amdgcn_ds_bpermute">, + ClangBuiltin<"__builtin_amdgcn_ds_bpermute">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn]>; // llvm.amdgcn.perm <src0> <src1> <selector> def int_amdgcn_perm : - GCCBuiltin<"__builtin_amdgcn_perm">, + ClangBuiltin<"__builtin_amdgcn_perm">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; //===----------------------------------------------------------------------===// +// GFX9 Intrinsics +//===----------------------------------------------------------------------===// + +class AMDGPUGlobalLoadLDS : Intrinsic < + [], + [LLVMQualPointerType<llvm_i8_ty, 1>, // Base global pointer to load from + LLVMQualPointerType<llvm_i8_ty, 3>, // LDS base pointer to store to + llvm_i32_ty, // Data byte size: 1/2/4 + llvm_i32_ty, // imm offset (applied to both global and LDS address) + llvm_i32_ty], // auxiliary data (imm, cachepolicy (bit 0 = glc/sc0, + // bit 1 = slc/sc1, + // bit 2 = dlc on gfx10+)) + // bit 4 = scc/nt on gfx90a+)) + [IntrWillReturn, NoCapture<ArgIndex<0>>, NoCapture<ArgIndex<1>>, + ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>], + "", [SDNPMemOperand]>; +def int_amdgcn_global_load_lds : AMDGPUGlobalLoadLDS; + +//===----------------------------------------------------------------------===// // GFX10 Intrinsics //===----------------------------------------------------------------------===// // llvm.amdgcn.permlane16 <old> <src0> <src1> <src2> <fi> <bound_control> -def int_amdgcn_permlane16 : GCCBuiltin<"__builtin_amdgcn_permlane16">, +def int_amdgcn_permlane16 : ClangBuiltin<"__builtin_amdgcn_permlane16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i1_ty], [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; // llvm.amdgcn.permlanex16 <old> <src0> <src1> <src2> <fi> <bound_control> -def int_amdgcn_permlanex16 : GCCBuiltin<"__builtin_amdgcn_permlanex16">, +def int_amdgcn_permlanex16 : ClangBuiltin<"__builtin_amdgcn_permlanex16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i1_ty], [IntrNoMem, IntrConvergent, IntrWillReturn, @@ -1789,9 +1939,9 @@ def int_amdgcn_mov_dpp8 : ImmArg<ArgIndex<1>>]>; def int_amdgcn_s_get_waveid_in_workgroup : - GCCBuiltin<"__builtin_amdgcn_s_get_waveid_in_workgroup">, + ClangBuiltin<"__builtin_amdgcn_s_get_waveid_in_workgroup">, Intrinsic<[llvm_i32_ty], [], - [IntrReadMem, IntrInaccessibleMemOnly, IntrWillReturn]>; + [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; class AMDGPUGlobalAtomicRtn<LLVMType vt> : Intrinsic < [vt], @@ -1813,13 +1963,82 @@ def int_amdgcn_image_bvh_intersect_ray : [IntrReadMem, IntrWillReturn]>; //===----------------------------------------------------------------------===// +// GFX11 Intrinsics +//===----------------------------------------------------------------------===// + +// llvm.amdgcn.permlane64 <src0> +def int_amdgcn_permlane64 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], + [IntrNoMem, IntrConvergent, IntrWillReturn]>; + +def int_amdgcn_ds_add_gs_reg_rtn : + ClangBuiltin<"__builtin_amdgcn_ds_add_gs_reg_rtn">, + Intrinsic<[llvm_anyint_ty], [llvm_i32_ty, llvm_i32_ty], + [ImmArg<ArgIndex<1>>, IntrHasSideEffects, IntrWillReturn]>; + +def int_amdgcn_ds_sub_gs_reg_rtn : + ClangBuiltin<"__builtin_amdgcn_ds_sub_gs_reg_rtn">, + Intrinsic<[llvm_anyint_ty], [llvm_i32_ty, llvm_i32_ty], + [ImmArg<ArgIndex<1>>, IntrHasSideEffects, IntrWillReturn]>; + +// WMMA (Wave Matrix Multiply-Accumulate) intrinsics +// +// These operations perform a matrix multiplication and accumulation of +// the form: D = A * B + C . + +class AMDGPUWmmaIntrinsic<LLVMType AB, LLVMType CD> : + Intrinsic< + [CD], // %D + [ + AB, // %A + AB, // %B + LLVMMatchType<0>, // %C + ], + [IntrNoMem, IntrConvergent, IntrWillReturn] +>; + +class AMDGPUWmmaIntrinsicOPSEL<LLVMType AB, LLVMType CD> : + Intrinsic< + [CD], // %D + [ + AB, // %A + AB, // %B + LLVMMatchType<0>, // %C + llvm_i1_ty, // %high + ], + [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg<ArgIndex<3>>] +>; + +class AMDGPUWmmaIntrinsicIU<LLVMType AB, LLVMType CD> : + Intrinsic< + [CD], // %D + [ + llvm_i1_ty, // %A_sign + AB, // %A + llvm_i1_ty, // %B_sign + AB, // %B + LLVMMatchType<0>, // %C + llvm_i1_ty, // %clamp + ], + [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>] +>; + +def int_amdgcn_wmma_f32_16x16x16_f16 : AMDGPUWmmaIntrinsic<llvm_v16f16_ty, llvm_anyfloat_ty>; +def int_amdgcn_wmma_f32_16x16x16_bf16 : AMDGPUWmmaIntrinsic<llvm_v16i16_ty, llvm_anyfloat_ty>; +def int_amdgcn_wmma_f16_16x16x16_f16 : AMDGPUWmmaIntrinsicOPSEL<llvm_v16f16_ty, llvm_anyfloat_ty>; +def int_amdgcn_wmma_bf16_16x16x16_bf16 : AMDGPUWmmaIntrinsicOPSEL<llvm_v16i16_ty, llvm_anyint_ty>; +def int_amdgcn_wmma_i32_16x16x16_iu8 : AMDGPUWmmaIntrinsicIU<llvm_v4i32_ty, llvm_anyint_ty>; +def int_amdgcn_wmma_i32_16x16x16_iu4 : AMDGPUWmmaIntrinsicIU<llvm_v2i32_ty, llvm_anyint_ty>; + + +//===----------------------------------------------------------------------===// // Deep learning intrinsics. //===----------------------------------------------------------------------===// // f32 %r = llvm.amdgcn.fdot2(v2f16 %a, v2f16 %b, f32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %c def int_amdgcn_fdot2 : - GCCBuiltin<"__builtin_amdgcn_fdot2">, + ClangBuiltin<"__builtin_amdgcn_fdot2">, Intrinsic< [llvm_float_ty], // %r [ @@ -1831,10 +2050,53 @@ def int_amdgcn_fdot2 : [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<3>>] >; +// f16 %r = llvm.amdgcn.fdot2.f16.f16(v2f16 %a, v2f16 %b, f16 %c) +// %r = %a[0] * %b[0] + %a[1] * %b[1] + %c +def int_amdgcn_fdot2_f16_f16 : + ClangBuiltin<"__builtin_amdgcn_fdot2_f16_f16">, + Intrinsic< + [llvm_half_ty], // %r + [ + llvm_v2f16_ty, // %a + llvm_v2f16_ty, // %b + llvm_half_ty // %c + ], + [IntrNoMem, IntrSpeculatable, IntrWillReturn] + >; + +// bf16 %r = llvm.amdgcn.fdot2.bf16.bf16(v2bf16 %a, v2bf16 %b, bf16 %c) +// %r = %a[0] * %b[0] + %a[1] * %b[1] + %c +def int_amdgcn_fdot2_bf16_bf16 : + ClangBuiltin<"__builtin_amdgcn_fdot2_bf16_bf16">, + Intrinsic< + [llvm_i16_ty], // %r + [ + llvm_v2i16_ty, // %a + llvm_v2i16_ty, // %b + llvm_i16_ty // %c + ], + [IntrNoMem, IntrSpeculatable, IntrWillReturn] + >; + +// f32 %r = llvm.amdgcn.fdot2.f32.bf16(v2bf16 %a, v2bf16 %b, f32 %c, i1 %clamp) +// %r = %a[0] * %b[0] + %a[1] * %b[1] + %c +def int_amdgcn_fdot2_f32_bf16 : + ClangBuiltin<"__builtin_amdgcn_fdot2_f32_bf16">, + Intrinsic< + [llvm_float_ty], // %r + [ + llvm_v2i16_ty, // %a + llvm_v2i16_ty, // %b + llvm_float_ty, // %c + llvm_i1_ty // %clamp + ], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<3>>] + >; + // i32 %r = llvm.amdgcn.sdot2(v2i16 %a, v2i16 %b, i32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %c def int_amdgcn_sdot2 : - GCCBuiltin<"__builtin_amdgcn_sdot2">, + ClangBuiltin<"__builtin_amdgcn_sdot2">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1849,7 +2111,7 @@ def int_amdgcn_sdot2 : // u32 %r = llvm.amdgcn.udot2(v2u16 %a, v2u16 %b, u32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %c def int_amdgcn_udot2 : - GCCBuiltin<"__builtin_amdgcn_udot2">, + ClangBuiltin<"__builtin_amdgcn_udot2">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1864,7 +2126,7 @@ def int_amdgcn_udot2 : // i32 %r = llvm.amdgcn.sdot4(v4i8 (as i32) %a, v4i8 (as i32) %b, i32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + %c def int_amdgcn_sdot4 : - GCCBuiltin<"__builtin_amdgcn_sdot4">, + ClangBuiltin<"__builtin_amdgcn_sdot4">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1879,7 +2141,7 @@ def int_amdgcn_sdot4 : // u32 %r = llvm.amdgcn.udot4(v4u8 (as u32) %a, v4u8 (as u32) %b, u32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + %c def int_amdgcn_udot4 : - GCCBuiltin<"__builtin_amdgcn_udot4">, + ClangBuiltin<"__builtin_amdgcn_udot4">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1891,11 +2153,32 @@ def int_amdgcn_udot4 : [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<3>>] >; +// i32 %r = llvm.amdgcn.sudot4(i1 %a_sign, v4i8 (as i32) %a, i1 %b_sign, v4i8 (as i32) %b, i32 %c, i1 %clamp) +// Treat input as signed (_sign = 1) or unsigned (_sign = 0). +// a[i in 0. . . 3] = (%a_sign ? a.i8[i] : promoteToSigned(a.u8[i])); +// b[i in 0. . . 3] = (%b_sign ? b.i8[i] : promoteToSigned(b.u8[i])); +// %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + %c +def int_amdgcn_sudot4 : + ClangBuiltin<"__builtin_amdgcn_sudot4">, + Intrinsic< + [llvm_i32_ty], // %r + [ + llvm_i1_ty, // %a_sign + llvm_i32_ty, // %a + llvm_i1_ty, // %b_sign + llvm_i32_ty, // %b + llvm_i32_ty, // %c + llvm_i1_ty // %clamp + ], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>] + >; + // i32 %r = llvm.amdgcn.sdot8(v8i4 (as i32) %a, v8i4 (as i32) %b, i32 %c, i1 %clamp) // %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + // %a[4] * %b[4] + %a[5] * %b[5] + %a[6] * %b[6] + %a[7] * %b[7] + %c def int_amdgcn_sdot8 : - GCCBuiltin<"__builtin_amdgcn_sdot8">, + ClangBuiltin<"__builtin_amdgcn_sdot8">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1911,7 +2194,7 @@ def int_amdgcn_sdot8 : // %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + // %a[4] * %b[4] + %a[5] * %b[5] + %a[6] * %b[6] + %a[7] * %b[7] + %c def int_amdgcn_udot8 : - GCCBuiltin<"__builtin_amdgcn_udot8">, + ClangBuiltin<"__builtin_amdgcn_udot8">, Intrinsic< [llvm_i32_ty], // %r [ @@ -1923,6 +2206,28 @@ def int_amdgcn_udot8 : [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<3>>] >; +// i32 %r = llvm.amdgcn.sudot8(i1 %a_sign, v8i4 (as i32) %a, i1 %b_sign, v8i4 (as i32) %b, i32 %c, i1 %clamp) +// Treat input as signed (_sign = 1) or unsigned (_sign = 0). +// a[i in 0. . . 7] = (%a_sign ? a.i4[i] : promoteToSigned(a.u4[i])); +// b[i in 0. . . 7] = (%b_sign ? b.i4[i] : promoteToSigned(b.u4[i])); +// %r = %a[0] * %b[0] + %a[1] * %b[1] + %a[2] * %b[2] + %a[3] * %b[3] + +// %a[4] * %b[4] + %a[5] * %b[5] + %a[6] * %b[6] + %a[7] * %b[7] + %c + def int_amdgcn_sudot8 : + ClangBuiltin<"__builtin_amdgcn_sudot8">, + Intrinsic< + [llvm_i32_ty], // %r + [ + llvm_i1_ty, // %a_sign + llvm_i32_ty, // %a + llvm_i1_ty, // %b_sign + llvm_i32_ty, // %b + llvm_i32_ty, // %c + llvm_i1_ty // %clamp + ], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>] + >; + //===----------------------------------------------------------------------===// // gfx908 intrinsics // ===----------------------------------------------------------------------===// @@ -1931,7 +2236,7 @@ def int_amdgcn_global_atomic_fadd : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; // llvm.amdgcn.mfma.*.* vdst, srcA, srcB, srcC, cbsz, abid, blgp class AMDGPUMfmaIntrinsic<LLVMType DestTy, LLVMType SrcABTy> : - GCCBuiltin<!subst("int", "__builtin", NAME)>, + ClangBuiltin<!subst("int", "__builtin", NAME)>, Intrinsic<[DestTy], [SrcABTy, SrcABTy, DestTy, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], @@ -1975,10 +2280,47 @@ def int_amdgcn_mfma_f32_4x4x4bf16_1k : AMDGPUMfmaIntrinsic<llvm_v4f32_ty, ll def int_amdgcn_mfma_f32_32x32x8bf16_1k : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4i16_ty>; def int_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUMfmaIntrinsic<llvm_v4f32_ty, llvm_v4i16_ty>; +// Note: in gfx940 BLGP argument is replaced by NEG bitfield in the DGEMM MFMA. +// Three bits corresponding to the neg modifier applied to the respective +// source operand. def int_amdgcn_mfma_f64_16x16x4f64 : AMDGPUMfmaIntrinsic<llvm_v4f64_ty, llvm_double_ty>; def int_amdgcn_mfma_f64_4x4x4f64 : AMDGPUMfmaIntrinsic<llvm_double_ty, llvm_double_ty>; //===----------------------------------------------------------------------===// +// gfx940 intrinsics +// ===----------------------------------------------------------------------===// + +// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. +def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUGlobalAtomicRtn<llvm_v2i16_ty>; +def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUGlobalAtomicRtn<llvm_v2i16_ty>; +def int_amdgcn_ds_fadd_v2bf16 : Intrinsic< + [llvm_v2i16_ty], + [LLVMQualPointerType<llvm_v2i16_ty, 3>, llvm_v2i16_ty], + [IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>]>, + ClangBuiltin<"__builtin_amdgcn_ds_atomic_fadd_v2bf16">; + +def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic<llvm_v4i32_ty, llvm_i64_ty>; +def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic<llvm_v16i32_ty, llvm_i64_ty>; +def int_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUMfmaIntrinsic<llvm_v4f32_ty, llvm_v2f32_ty>; +def int_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v2f32_ty>; + +// llvm.amdgcn.smfmac.?32.* vdst, srcA, srcB, srcC, index, cbsz, abid +class AMDGPUMSmfmacIntrinsic<LLVMType DestTy, LLVMType SrcA, LLVMType SrcB> : + ClangBuiltin<!subst("int", "__builtin", NAME)>, + Intrinsic<[DestTy], + [SrcA, SrcB, DestTy, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, IntrWillReturn, + ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; + +def int_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUMSmfmacIntrinsic<llvm_v4f32_ty, llvm_v4f16_ty, llvm_v8f16_ty>; +def int_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUMSmfmacIntrinsic<llvm_v16f32_ty, llvm_v4f16_ty, llvm_v8f16_ty>; +def int_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUMSmfmacIntrinsic<llvm_v4f32_ty, llvm_v4i16_ty, llvm_v8i16_ty>; +def int_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUMSmfmacIntrinsic<llvm_v16f32_ty, llvm_v4i16_ty, llvm_v8i16_ty>; +def int_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUMSmfmacIntrinsic<llvm_v4i32_ty, llvm_v2i32_ty, llvm_v4i32_ty>; +def int_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUMSmfmacIntrinsic<llvm_v16i32_ty, llvm_v2i32_ty, llvm_v4i32_ty>; + +//===----------------------------------------------------------------------===// // Special Intrinsics for backend internal use only. No frontend // should emit calls to these. // ===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td index a42484757592..3d905dbca6b9 100644 --- a/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/llvm/include/llvm/IR/IntrinsicsARM.td @@ -22,199 +22,199 @@ let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". def int_arm_space : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; // 16-bit multiplications -def int_arm_smulbb : GCCBuiltin<"__builtin_arm_smulbb">, +def int_arm_smulbb : ClangBuiltin<"__builtin_arm_smulbb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smulbt : GCCBuiltin<"__builtin_arm_smulbt">, +def int_arm_smulbt : ClangBuiltin<"__builtin_arm_smulbt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smultb : GCCBuiltin<"__builtin_arm_smultb">, +def int_arm_smultb : ClangBuiltin<"__builtin_arm_smultb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smultt : GCCBuiltin<"__builtin_arm_smultt">, +def int_arm_smultt : ClangBuiltin<"__builtin_arm_smultt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smulwb : GCCBuiltin<"__builtin_arm_smulwb">, +def int_arm_smulwb : ClangBuiltin<"__builtin_arm_smulwb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smulwt : GCCBuiltin<"__builtin_arm_smulwt">, +def int_arm_smulwt : ClangBuiltin<"__builtin_arm_smulwt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// // Saturating Arithmetic -def int_arm_qadd : GCCBuiltin<"__builtin_arm_qadd">, +def int_arm_qadd : ClangBuiltin<"__builtin_arm_qadd">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [Commutative, IntrNoMem]>; -def int_arm_qsub : GCCBuiltin<"__builtin_arm_qsub">, +def int_arm_qsub : ClangBuiltin<"__builtin_arm_qsub">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_ssat : GCCBuiltin<"__builtin_arm_ssat">, +def int_arm_ssat : ClangBuiltin<"__builtin_arm_ssat">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_usat : GCCBuiltin<"__builtin_arm_usat">, +def int_arm_usat : ClangBuiltin<"__builtin_arm_usat">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Accumulating multiplications -def int_arm_smlabb : GCCBuiltin<"__builtin_arm_smlabb">, +def int_arm_smlabb : ClangBuiltin<"__builtin_arm_smlabb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlabt : GCCBuiltin<"__builtin_arm_smlabt">, +def int_arm_smlabt : ClangBuiltin<"__builtin_arm_smlabt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlatb : GCCBuiltin<"__builtin_arm_smlatb">, +def int_arm_smlatb : ClangBuiltin<"__builtin_arm_smlatb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlatt : GCCBuiltin<"__builtin_arm_smlatt">, +def int_arm_smlatt : ClangBuiltin<"__builtin_arm_smlatt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlawb : GCCBuiltin<"__builtin_arm_smlawb">, +def int_arm_smlawb : ClangBuiltin<"__builtin_arm_smlawb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlawt : GCCBuiltin<"__builtin_arm_smlawt">, +def int_arm_smlawt : ClangBuiltin<"__builtin_arm_smlawt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Parallel 16-bit saturation -def int_arm_ssat16 : GCCBuiltin<"__builtin_arm_ssat16">, +def int_arm_ssat16 : ClangBuiltin<"__builtin_arm_ssat16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_usat16 : GCCBuiltin<"__builtin_arm_usat16">, +def int_arm_usat16 : ClangBuiltin<"__builtin_arm_usat16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Packing and unpacking -def int_arm_sxtab16 : GCCBuiltin<"__builtin_arm_sxtab16">, +def int_arm_sxtab16 : ClangBuiltin<"__builtin_arm_sxtab16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_sxtb16 : GCCBuiltin<"__builtin_arm_sxtb16">, +def int_arm_sxtb16 : ClangBuiltin<"__builtin_arm_sxtb16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_arm_uxtab16 : GCCBuiltin<"__builtin_arm_uxtab16">, +def int_arm_uxtab16 : ClangBuiltin<"__builtin_arm_uxtab16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uxtb16 : GCCBuiltin<"__builtin_arm_uxtb16">, +def int_arm_uxtb16 : ClangBuiltin<"__builtin_arm_uxtb16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; // Parallel selection, reads the GE flags. -def int_arm_sel : GCCBuiltin<"__builtin_arm_sel">, +def int_arm_sel : ClangBuiltin<"__builtin_arm_sel">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>; // Parallel 8-bit addition and subtraction -def int_arm_qadd8 : GCCBuiltin<"__builtin_arm_qadd8">, +def int_arm_qadd8 : ClangBuiltin<"__builtin_arm_qadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_qsub8 : GCCBuiltin<"__builtin_arm_qsub8">, +def int_arm_qsub8 : ClangBuiltin<"__builtin_arm_qsub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_sadd8 : GCCBuiltin<"__builtin_arm_sadd8">, +def int_arm_sadd8 : ClangBuiltin<"__builtin_arm_sadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; -def int_arm_shadd8 : GCCBuiltin<"__builtin_arm_shadd8">, +def int_arm_shadd8 : ClangBuiltin<"__builtin_arm_shadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_shsub8 : GCCBuiltin<"__builtin_arm_shsub8">, +def int_arm_shsub8 : ClangBuiltin<"__builtin_arm_shsub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_ssub8 : GCCBuiltin<"__builtin_arm_ssub8">, +def int_arm_ssub8 : ClangBuiltin<"__builtin_arm_ssub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_uadd8 : GCCBuiltin<"__builtin_arm_uadd8">, +def int_arm_uadd8 : ClangBuiltin<"__builtin_arm_uadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; -def int_arm_uhadd8 : GCCBuiltin<"__builtin_arm_uhadd8">, +def int_arm_uhadd8 : ClangBuiltin<"__builtin_arm_uhadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uhsub8 : GCCBuiltin<"__builtin_arm_uhsub8">, +def int_arm_uhsub8 : ClangBuiltin<"__builtin_arm_uhsub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqadd8 : GCCBuiltin<"__builtin_arm_uqadd8">, +def int_arm_uqadd8 : ClangBuiltin<"__builtin_arm_uqadd8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqsub8 : GCCBuiltin<"__builtin_arm_uqsub8">, +def int_arm_uqsub8 : ClangBuiltin<"__builtin_arm_uqsub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_usub8 : GCCBuiltin<"__builtin_arm_usub8">, +def int_arm_usub8 : ClangBuiltin<"__builtin_arm_usub8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Sum of 8-bit absolute differences -def int_arm_usad8 : GCCBuiltin<"__builtin_arm_usad8">, +def int_arm_usad8 : ClangBuiltin<"__builtin_arm_usad8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_usada8 : GCCBuiltin<"__builtin_arm_usada8">, +def int_arm_usada8 : ClangBuiltin<"__builtin_arm_usada8">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Parallel 16-bit addition and subtraction -def int_arm_qadd16 : GCCBuiltin<"__builtin_arm_qadd16">, +def int_arm_qadd16 : ClangBuiltin<"__builtin_arm_qadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_qasx : GCCBuiltin<"__builtin_arm_qasx">, +def int_arm_qasx : ClangBuiltin<"__builtin_arm_qasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_qsax : GCCBuiltin<"__builtin_arm_qsax">, +def int_arm_qsax : ClangBuiltin<"__builtin_arm_qsax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_qsub16 : GCCBuiltin<"__builtin_arm_qsub16">, +def int_arm_qsub16 : ClangBuiltin<"__builtin_arm_qsub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_sadd16 : GCCBuiltin<"__builtin_arm_sadd16">, +def int_arm_sadd16 : ClangBuiltin<"__builtin_arm_sadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_sasx : GCCBuiltin<"__builtin_arm_sasx">, +def int_arm_sasx : ClangBuiltin<"__builtin_arm_sasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; -def int_arm_shadd16 : GCCBuiltin<"__builtin_arm_shadd16">, +def int_arm_shadd16 : ClangBuiltin<"__builtin_arm_shadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_shasx : GCCBuiltin<"__builtin_arm_shasx">, +def int_arm_shasx : ClangBuiltin<"__builtin_arm_shasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_shsax : GCCBuiltin<"__builtin_arm_shsax">, +def int_arm_shsax : ClangBuiltin<"__builtin_arm_shsax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_shsub16 : GCCBuiltin<"__builtin_arm_shsub16">, +def int_arm_shsub16 : ClangBuiltin<"__builtin_arm_shsub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_ssax : GCCBuiltin<"__builtin_arm_ssax">, +def int_arm_ssax : ClangBuiltin<"__builtin_arm_ssax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_ssub16 : GCCBuiltin<"__builtin_arm_ssub16">, +def int_arm_ssub16 : ClangBuiltin<"__builtin_arm_ssub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_uadd16 : GCCBuiltin<"__builtin_arm_uadd16">, +def int_arm_uadd16 : ClangBuiltin<"__builtin_arm_uadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_uasx : GCCBuiltin<"__builtin_arm_uasx">, +def int_arm_uasx : ClangBuiltin<"__builtin_arm_uasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; -def int_arm_uhadd16 : GCCBuiltin<"__builtin_arm_uhadd16">, +def int_arm_uhadd16 : ClangBuiltin<"__builtin_arm_uhadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uhasx : GCCBuiltin<"__builtin_arm_uhasx">, +def int_arm_uhasx : ClangBuiltin<"__builtin_arm_uhasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uhsax : GCCBuiltin<"__builtin_arm_uhsax">, +def int_arm_uhsax : ClangBuiltin<"__builtin_arm_uhsax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uhsub16 : GCCBuiltin<"__builtin_arm_uhsub16">, +def int_arm_uhsub16 : ClangBuiltin<"__builtin_arm_uhsub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqadd16 : GCCBuiltin<"__builtin_arm_uqadd16">, +def int_arm_uqadd16 : ClangBuiltin<"__builtin_arm_uqadd16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqasx : GCCBuiltin<"__builtin_arm_uqasx">, +def int_arm_uqasx : ClangBuiltin<"__builtin_arm_uqasx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqsax : GCCBuiltin<"__builtin_arm_uqsax">, +def int_arm_uqsax : ClangBuiltin<"__builtin_arm_uqsax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_uqsub16 : GCCBuiltin<"__builtin_arm_uqsub16">, +def int_arm_uqsub16 : ClangBuiltin<"__builtin_arm_uqsub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Writes to the GE bits. -def int_arm_usax : GCCBuiltin<"__builtin_arm_usax">, +def int_arm_usax : ClangBuiltin<"__builtin_arm_usax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Writes to the GE bits. -def int_arm_usub16 : GCCBuiltin<"__builtin_arm_usub16">, +def int_arm_usub16 : ClangBuiltin<"__builtin_arm_usub16">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; // Parallel 16-bit multiplication -def int_arm_smlad : GCCBuiltin<"__builtin_arm_smlad">, +def int_arm_smlad : ClangBuiltin<"__builtin_arm_smlad">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smladx : GCCBuiltin<"__builtin_arm_smladx">, +def int_arm_smladx : ClangBuiltin<"__builtin_arm_smladx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlald : GCCBuiltin<"__builtin_arm_smlald">, +def int_arm_smlald : ClangBuiltin<"__builtin_arm_smlald">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; -def int_arm_smlaldx : GCCBuiltin<"__builtin_arm_smlaldx">, +def int_arm_smlaldx : ClangBuiltin<"__builtin_arm_smlaldx">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; -def int_arm_smlsd : GCCBuiltin<"__builtin_arm_smlsd">, +def int_arm_smlsd : ClangBuiltin<"__builtin_arm_smlsd">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlsdx : GCCBuiltin<"__builtin_arm_smlsdx">, +def int_arm_smlsdx : ClangBuiltin<"__builtin_arm_smlsdx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smlsld : GCCBuiltin<"__builtin_arm_smlsld">, +def int_arm_smlsld : ClangBuiltin<"__builtin_arm_smlsld">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; -def int_arm_smlsldx : GCCBuiltin<"__builtin_arm_smlsldx">, +def int_arm_smlsldx : ClangBuiltin<"__builtin_arm_smlsldx">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; -def int_arm_smuad : GCCBuiltin<"__builtin_arm_smuad">, +def int_arm_smuad : ClangBuiltin<"__builtin_arm_smuad">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smuadx : GCCBuiltin<"__builtin_arm_smuadx">, +def int_arm_smuadx : ClangBuiltin<"__builtin_arm_smuadx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smusd : GCCBuiltin<"__builtin_arm_smusd">, +def int_arm_smusd : ClangBuiltin<"__builtin_arm_smusd">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_arm_smusdx : GCCBuiltin<"__builtin_arm_smusdx">, +def int_arm_smusdx : ClangBuiltin<"__builtin_arm_smusdx">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; @@ -239,19 +239,19 @@ def int_arm_ldaexd : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_ptr_ty]>; //===----------------------------------------------------------------------===// // Data barrier instructions -def int_arm_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, +def int_arm_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>; -def int_arm_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, +def int_arm_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>; -def int_arm_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, +def int_arm_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>; //===----------------------------------------------------------------------===// // VFP -def int_arm_get_fpscr : GCCBuiltin<"__builtin_arm_get_fpscr">, +def int_arm_get_fpscr : ClangBuiltin<"__builtin_arm_get_fpscr">, Intrinsic<[llvm_i32_ty], [], []>; -def int_arm_set_fpscr : GCCBuiltin<"__builtin_arm_set_fpscr">, +def int_arm_set_fpscr : ClangBuiltin<"__builtin_arm_set_fpscr">, Intrinsic<[], [llvm_i32_ty], []>; def int_arm_vcvtr : Intrinsic<[llvm_float_ty], [llvm_anyfloat_ty], [IntrNoMem]>; @@ -261,47 +261,47 @@ def int_arm_vcvtru : Intrinsic<[llvm_float_ty], [llvm_anyfloat_ty], //===----------------------------------------------------------------------===// // Coprocessor -def int_arm_ldc : GCCBuiltin<"__builtin_arm_ldc">, +def int_arm_ldc : ClangBuiltin<"__builtin_arm_ldc">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_ldcl : GCCBuiltin<"__builtin_arm_ldcl">, +def int_arm_ldcl : ClangBuiltin<"__builtin_arm_ldcl">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_ldc2 : GCCBuiltin<"__builtin_arm_ldc2">, +def int_arm_ldc2 : ClangBuiltin<"__builtin_arm_ldc2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_ldc2l : GCCBuiltin<"__builtin_arm_ldc2l">, +def int_arm_ldc2l : ClangBuiltin<"__builtin_arm_ldc2l">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_stc : GCCBuiltin<"__builtin_arm_stc">, +def int_arm_stc : ClangBuiltin<"__builtin_arm_stc">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_stcl : GCCBuiltin<"__builtin_arm_stcl">, +def int_arm_stcl : ClangBuiltin<"__builtin_arm_stcl">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_stc2 : GCCBuiltin<"__builtin_arm_stc2">, +def int_arm_stc2 : ClangBuiltin<"__builtin_arm_stc2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; -def int_arm_stc2l : GCCBuiltin<"__builtin_arm_stc2l">, +def int_arm_stc2l : ClangBuiltin<"__builtin_arm_stc2l">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; // Move to coprocessor -def int_arm_mcr : GCCBuiltin<"__builtin_arm_mcr">, +def int_arm_mcr : ClangBuiltin<"__builtin_arm_mcr">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; -def int_arm_mcr2 : GCCBuiltin<"__builtin_arm_mcr2">, +def int_arm_mcr2 : ClangBuiltin<"__builtin_arm_mcr2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; // Move from coprocessor -def int_arm_mrc : GCCBuiltin<"__builtin_arm_mrc">, +def int_arm_mrc : ClangBuiltin<"__builtin_arm_mrc">, MSBuiltin<"_MoveFromCoprocessor">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; -def int_arm_mrc2 : GCCBuiltin<"__builtin_arm_mrc2">, +def int_arm_mrc2 : ClangBuiltin<"__builtin_arm_mrc2">, MSBuiltin<"_MoveFromCoprocessor2">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; // Coprocessor data processing -def int_arm_cdp : GCCBuiltin<"__builtin_arm_cdp">, +def int_arm_cdp : ClangBuiltin<"__builtin_arm_cdp">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; -def int_arm_cdp2 : GCCBuiltin<"__builtin_arm_cdp2">, +def int_arm_cdp2 : ClangBuiltin<"__builtin_arm_cdp2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; @@ -335,13 +335,13 @@ def int_arm_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], //===----------------------------------------------------------------------===// // CMSE -def int_arm_cmse_tt : GCCBuiltin<"__builtin_arm_cmse_TT">, +def int_arm_cmse_tt : ClangBuiltin<"__builtin_arm_cmse_TT">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; -def int_arm_cmse_ttt : GCCBuiltin<"__builtin_arm_cmse_TTT">, +def int_arm_cmse_ttt : ClangBuiltin<"__builtin_arm_cmse_TTT">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; -def int_arm_cmse_tta : GCCBuiltin<"__builtin_arm_cmse_TTA">, +def int_arm_cmse_tta : ClangBuiltin<"__builtin_arm_cmse_TTA">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; -def int_arm_cmse_ttat : GCCBuiltin<"__builtin_arm_cmse_TTAT">, +def int_arm_cmse_ttat : ClangBuiltin<"__builtin_arm_cmse_TTAT">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// @@ -1158,7 +1158,7 @@ defm int_arm_mve_vabav: MVEPredicated< [llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], llvm_anyvector_ty>; -// The following 3 instrinsics are MVE vector reductions with two vector +// The following 3 intrinsics are MVE vector reductions with two vector // operands. // The first 3 operands are boolean flags (must be compile-time constants): // * unsigned - the instruction operates on vectors of unsigned values and diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td index a6bd6f841aab..8916b60d2be3 100644 --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -12,29 +12,29 @@ // Specialized loads from packet let TargetPrefix = "bpf" in { // All intrinsics start with "llvm.bpf." - def int_bpf_load_byte : GCCBuiltin<"__builtin_bpf_load_byte">, + def int_bpf_load_byte : ClangBuiltin<"__builtin_bpf_load_byte">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; - def int_bpf_load_half : GCCBuiltin<"__builtin_bpf_load_half">, + def int_bpf_load_half : ClangBuiltin<"__builtin_bpf_load_half">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; - def int_bpf_load_word : GCCBuiltin<"__builtin_bpf_load_word">, + def int_bpf_load_word : ClangBuiltin<"__builtin_bpf_load_word">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; - def int_bpf_pseudo : GCCBuiltin<"__builtin_bpf_pseudo">, + def int_bpf_pseudo : ClangBuiltin<"__builtin_bpf_pseudo">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>; - def int_bpf_preserve_field_info : GCCBuiltin<"__builtin_bpf_preserve_field_info">, + def int_bpf_preserve_field_info : ClangBuiltin<"__builtin_bpf_preserve_field_info">, Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i64_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_bpf_btf_type_id : GCCBuiltin<"__builtin_bpf_btf_type_id">, + def int_bpf_btf_type_id : ClangBuiltin<"__builtin_bpf_btf_type_id">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; - def int_bpf_preserve_type_info : GCCBuiltin<"__builtin_bpf_preserve_type_info">, + def int_bpf_preserve_type_info : ClangBuiltin<"__builtin_bpf_preserve_type_info">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; - def int_bpf_preserve_enum_value : GCCBuiltin<"__builtin_bpf_preserve_enum_value">, + def int_bpf_preserve_enum_value : ClangBuiltin<"__builtin_bpf_preserve_enum_value">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_ptr_ty, llvm_i64_ty], [IntrNoMem]>; - def int_bpf_passthrough : GCCBuiltin<"__builtin_bpf_passthrough">, + def int_bpf_passthrough : ClangBuiltin<"__builtin_bpf_passthrough">, Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]>; - def int_bpf_compare : GCCBuiltin<"__builtin_bpf_compare">, + def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">, Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty], [IntrNoMem]>; } diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td new file mode 100644 index 000000000000..4a21cf1eb7fc --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -0,0 +1,20 @@ +//===- IntrinsicsDirectX.td - Defines DirectX intrinsics ---*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines all of the DirectX-specific intrinsics. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "dxil" in { + +def int_dxil_thread_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; +def int_dxil_group_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; +def int_dxil_thread_id_in_group : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; +def int_dxil_flattened_thread_id_in_group : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrWillReturn]>; + +} diff --git a/llvm/include/llvm/IR/IntrinsicsHexagon.td b/llvm/include/llvm/IR/IntrinsicsHexagon.td index 212262c28706..52c29ef31f0a 100644 --- a/llvm/include/llvm/IR/IntrinsicsHexagon.td +++ b/llvm/include/llvm/IR/IntrinsicsHexagon.td @@ -18,7 +18,7 @@ let TargetPrefix = "hexagon" in { class Hexagon_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types, list<LLVMType> param_types, list<IntrinsicProperty> properties> - : GCCBuiltin<!strconcat("__builtin_", GCCIntSuffix)>, + : ClangBuiltin<!strconcat("__builtin_", GCCIntSuffix)>, Intrinsic<ret_types, param_types, properties>; /// Hexagon_NonGCC_Intrinsic - Base class for bitcode convertible Hexagon @@ -404,4 +404,15 @@ def int_hexagon_V6_vmaskedstorenq_128B: Hexagon_custom_vms_Intrinsic_128B; def int_hexagon_V6_vmaskedstorentq_128B: Hexagon_custom_vms_Intrinsic_128B; def int_hexagon_V6_vmaskedstorentnq_128B: Hexagon_custom_vms_Intrinsic_128B; + +// Intrinsic for instrumentation based profiling using a custom handler. The +// name of the handler is passed as the first operand to the intrinsic. The +// handler can take only one int32 input which is passed as the second +// operand to the intrinsic. +def int_hexagon_instrprof_custom + : Hexagon_NonGCC_Intrinsic<[], + [llvm_ptr_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly]>; + + include "llvm/IR/IntrinsicsHexagonDep.td" diff --git a/llvm/include/llvm/IR/IntrinsicsMips.td b/llvm/include/llvm/IR/IntrinsicsMips.td index 271142ca7788..3056f37b9d87 100644 --- a/llvm/include/llvm/IR/IntrinsicsMips.td +++ b/llvm/include/llvm/IR/IntrinsicsMips.td @@ -24,370 +24,370 @@ let TargetPrefix = "mips" in { // All intrinsics start with "llvm.mips.". //===----------------------------------------------------------------------===// // Addition/subtraction -def int_mips_addu_qb : GCCBuiltin<"__builtin_mips_addu_qb">, +def int_mips_addu_qb : ClangBuiltin<"__builtin_mips_addu_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative, IntrNoMem]>; -def int_mips_addu_s_qb : GCCBuiltin<"__builtin_mips_addu_s_qb">, +def int_mips_addu_s_qb : ClangBuiltin<"__builtin_mips_addu_s_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative, IntrNoMem]>; -def int_mips_subu_qb : GCCBuiltin<"__builtin_mips_subu_qb">, +def int_mips_subu_qb : ClangBuiltin<"__builtin_mips_subu_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_subu_s_qb : GCCBuiltin<"__builtin_mips_subu_s_qb">, +def int_mips_subu_s_qb : ClangBuiltin<"__builtin_mips_subu_s_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_addq_ph : GCCBuiltin<"__builtin_mips_addq_ph">, +def int_mips_addq_ph : ClangBuiltin<"__builtin_mips_addq_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative, IntrNoMem]>; -def int_mips_addq_s_ph : GCCBuiltin<"__builtin_mips_addq_s_ph">, +def int_mips_addq_s_ph : ClangBuiltin<"__builtin_mips_addq_s_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative, IntrNoMem]>; -def int_mips_subq_ph : GCCBuiltin<"__builtin_mips_subq_ph">, +def int_mips_subq_ph : ClangBuiltin<"__builtin_mips_subq_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_subq_s_ph : GCCBuiltin<"__builtin_mips_subq_s_ph">, +def int_mips_subq_s_ph : ClangBuiltin<"__builtin_mips_subq_s_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_madd: GCCBuiltin<"__builtin_mips_madd">, +def int_mips_madd: ClangBuiltin<"__builtin_mips_madd">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; -def int_mips_maddu: GCCBuiltin<"__builtin_mips_maddu">, +def int_mips_maddu: ClangBuiltin<"__builtin_mips_maddu">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; -def int_mips_msub: GCCBuiltin<"__builtin_mips_msub">, +def int_mips_msub: ClangBuiltin<"__builtin_mips_msub">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_msubu: GCCBuiltin<"__builtin_mips_msubu">, +def int_mips_msubu: ClangBuiltin<"__builtin_mips_msubu">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_addq_s_w: GCCBuiltin<"__builtin_mips_addq_s_w">, +def int_mips_addq_s_w: ClangBuiltin<"__builtin_mips_addq_s_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [Commutative]>; -def int_mips_subq_s_w: GCCBuiltin<"__builtin_mips_subq_s_w">, +def int_mips_subq_s_w: ClangBuiltin<"__builtin_mips_subq_s_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], []>; -def int_mips_addsc: GCCBuiltin<"__builtin_mips_addsc">, +def int_mips_addsc: ClangBuiltin<"__builtin_mips_addsc">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [Commutative]>; -def int_mips_addwc: GCCBuiltin<"__builtin_mips_addwc">, +def int_mips_addwc: ClangBuiltin<"__builtin_mips_addwc">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [Commutative]>; -def int_mips_modsub: GCCBuiltin<"__builtin_mips_modsub">, +def int_mips_modsub: ClangBuiltin<"__builtin_mips_modsub">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_raddu_w_qb: GCCBuiltin<"__builtin_mips_raddu_w_qb">, +def int_mips_raddu_w_qb: ClangBuiltin<"__builtin_mips_raddu_w_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// // Absolute value -def int_mips_absq_s_ph: GCCBuiltin<"__builtin_mips_absq_s_ph">, +def int_mips_absq_s_ph: ClangBuiltin<"__builtin_mips_absq_s_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty], []>; -def int_mips_absq_s_w: GCCBuiltin<"__builtin_mips_absq_s_w">, +def int_mips_absq_s_w: ClangBuiltin<"__builtin_mips_absq_s_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty], []>; //===----------------------------------------------------------------------===// // Precision reduce/expand -def int_mips_precrq_qb_ph: GCCBuiltin<"__builtin_mips_precrq_qb_ph">, +def int_mips_precrq_qb_ph: ClangBuiltin<"__builtin_mips_precrq_qb_ph">, Intrinsic<[llvm_v4i8_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_precrqu_s_qb_ph: GCCBuiltin<"__builtin_mips_precrqu_s_qb_ph">, +def int_mips_precrqu_s_qb_ph: ClangBuiltin<"__builtin_mips_precrqu_s_qb_ph">, Intrinsic<[llvm_v4i8_ty], [mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_precrq_ph_w: GCCBuiltin<"__builtin_mips_precrq_ph_w">, +def int_mips_precrq_ph_w: ClangBuiltin<"__builtin_mips_precrq_ph_w">, Intrinsic<[mips_v2q15_ty], [mips_q31_ty, mips_q31_ty], [IntrNoMem]>; -def int_mips_precrq_rs_ph_w: GCCBuiltin<"__builtin_mips_precrq_rs_ph_w">, +def int_mips_precrq_rs_ph_w: ClangBuiltin<"__builtin_mips_precrq_rs_ph_w">, Intrinsic<[mips_v2q15_ty], [mips_q31_ty, mips_q31_ty], []>; -def int_mips_preceq_w_phl: GCCBuiltin<"__builtin_mips_preceq_w_phl">, +def int_mips_preceq_w_phl: ClangBuiltin<"__builtin_mips_preceq_w_phl">, Intrinsic<[mips_q31_ty], [mips_v2q15_ty], [IntrNoMem]>; -def int_mips_preceq_w_phr: GCCBuiltin<"__builtin_mips_preceq_w_phr">, +def int_mips_preceq_w_phr: ClangBuiltin<"__builtin_mips_preceq_w_phr">, Intrinsic<[mips_q31_ty], [mips_v2q15_ty], [IntrNoMem]>; -def int_mips_precequ_ph_qbl: GCCBuiltin<"__builtin_mips_precequ_ph_qbl">, +def int_mips_precequ_ph_qbl: ClangBuiltin<"__builtin_mips_precequ_ph_qbl">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_precequ_ph_qbr: GCCBuiltin<"__builtin_mips_precequ_ph_qbr">, +def int_mips_precequ_ph_qbr: ClangBuiltin<"__builtin_mips_precequ_ph_qbr">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_precequ_ph_qbla: GCCBuiltin<"__builtin_mips_precequ_ph_qbla">, +def int_mips_precequ_ph_qbla: ClangBuiltin<"__builtin_mips_precequ_ph_qbla">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_precequ_ph_qbra: GCCBuiltin<"__builtin_mips_precequ_ph_qbra">, +def int_mips_precequ_ph_qbra: ClangBuiltin<"__builtin_mips_precequ_ph_qbra">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_preceu_ph_qbl: GCCBuiltin<"__builtin_mips_preceu_ph_qbl">, +def int_mips_preceu_ph_qbl: ClangBuiltin<"__builtin_mips_preceu_ph_qbl">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_preceu_ph_qbr: GCCBuiltin<"__builtin_mips_preceu_ph_qbr">, +def int_mips_preceu_ph_qbr: ClangBuiltin<"__builtin_mips_preceu_ph_qbr">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_preceu_ph_qbla: GCCBuiltin<"__builtin_mips_preceu_ph_qbla">, +def int_mips_preceu_ph_qbla: ClangBuiltin<"__builtin_mips_preceu_ph_qbla">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_preceu_ph_qbra: GCCBuiltin<"__builtin_mips_preceu_ph_qbra">, +def int_mips_preceu_ph_qbra: ClangBuiltin<"__builtin_mips_preceu_ph_qbra">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// // Shift -def int_mips_shll_qb: GCCBuiltin<"__builtin_mips_shll_qb">, +def int_mips_shll_qb: ClangBuiltin<"__builtin_mips_shll_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_i32_ty], []>; -def int_mips_shrl_qb: GCCBuiltin<"__builtin_mips_shrl_qb">, +def int_mips_shrl_qb: ClangBuiltin<"__builtin_mips_shrl_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shll_ph: GCCBuiltin<"__builtin_mips_shll_ph">, +def int_mips_shll_ph: ClangBuiltin<"__builtin_mips_shll_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, llvm_i32_ty], []>; -def int_mips_shll_s_ph: GCCBuiltin<"__builtin_mips_shll_s_ph">, +def int_mips_shll_s_ph: ClangBuiltin<"__builtin_mips_shll_s_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, llvm_i32_ty], []>; -def int_mips_shra_ph: GCCBuiltin<"__builtin_mips_shra_ph">, +def int_mips_shra_ph: ClangBuiltin<"__builtin_mips_shra_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shra_r_ph: GCCBuiltin<"__builtin_mips_shra_r_ph">, +def int_mips_shra_r_ph: ClangBuiltin<"__builtin_mips_shra_r_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shll_s_w: GCCBuiltin<"__builtin_mips_shll_s_w">, +def int_mips_shll_s_w: ClangBuiltin<"__builtin_mips_shll_s_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, llvm_i32_ty], []>; -def int_mips_shra_r_w: GCCBuiltin<"__builtin_mips_shra_r_w">, +def int_mips_shra_r_w: ClangBuiltin<"__builtin_mips_shra_r_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shilo: GCCBuiltin<"__builtin_mips_shilo">, +def int_mips_shilo: ClangBuiltin<"__builtin_mips_shilo">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// // Multiplication -def int_mips_muleu_s_ph_qbl: GCCBuiltin<"__builtin_mips_muleu_s_ph_qbl">, +def int_mips_muleu_s_ph_qbl: ClangBuiltin<"__builtin_mips_muleu_s_ph_qbl">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty, mips_v2q15_ty], []>; -def int_mips_muleu_s_ph_qbr: GCCBuiltin<"__builtin_mips_muleu_s_ph_qbr">, +def int_mips_muleu_s_ph_qbr: ClangBuiltin<"__builtin_mips_muleu_s_ph_qbr">, Intrinsic<[mips_v2q15_ty], [llvm_v4i8_ty, mips_v2q15_ty], []>; -def int_mips_mulq_rs_ph: GCCBuiltin<"__builtin_mips_mulq_rs_ph">, +def int_mips_mulq_rs_ph: ClangBuiltin<"__builtin_mips_mulq_rs_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative]>; -def int_mips_muleq_s_w_phl: GCCBuiltin<"__builtin_mips_muleq_s_w_phl">, +def int_mips_muleq_s_w_phl: ClangBuiltin<"__builtin_mips_muleq_s_w_phl">, Intrinsic<[mips_q31_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative]>; -def int_mips_muleq_s_w_phr: GCCBuiltin<"__builtin_mips_muleq_s_w_phr">, +def int_mips_muleq_s_w_phr: ClangBuiltin<"__builtin_mips_muleq_s_w_phr">, Intrinsic<[mips_q31_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative]>; -def int_mips_mulsaq_s_w_ph: GCCBuiltin<"__builtin_mips_mulsaq_s_w_ph">, +def int_mips_mulsaq_s_w_ph: ClangBuiltin<"__builtin_mips_mulsaq_s_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_maq_s_w_phl: GCCBuiltin<"__builtin_mips_maq_s_w_phl">, +def int_mips_maq_s_w_phl: ClangBuiltin<"__builtin_mips_maq_s_w_phl">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_maq_s_w_phr: GCCBuiltin<"__builtin_mips_maq_s_w_phr">, +def int_mips_maq_s_w_phr: ClangBuiltin<"__builtin_mips_maq_s_w_phr">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_maq_sa_w_phl: GCCBuiltin<"__builtin_mips_maq_sa_w_phl">, +def int_mips_maq_sa_w_phl: ClangBuiltin<"__builtin_mips_maq_sa_w_phl">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_maq_sa_w_phr: GCCBuiltin<"__builtin_mips_maq_sa_w_phr">, +def int_mips_maq_sa_w_phr: ClangBuiltin<"__builtin_mips_maq_sa_w_phr">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_mult: GCCBuiltin<"__builtin_mips_mult">, +def int_mips_mult: ClangBuiltin<"__builtin_mips_mult">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; -def int_mips_multu: GCCBuiltin<"__builtin_mips_multu">, +def int_mips_multu: ClangBuiltin<"__builtin_mips_multu">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; //===----------------------------------------------------------------------===// // Dot product with accumulate/subtract -def int_mips_dpau_h_qbl: GCCBuiltin<"__builtin_mips_dpau_h_qbl">, +def int_mips_dpau_h_qbl: ClangBuiltin<"__builtin_mips_dpau_h_qbl">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_dpau_h_qbr: GCCBuiltin<"__builtin_mips_dpau_h_qbr">, +def int_mips_dpau_h_qbr: ClangBuiltin<"__builtin_mips_dpau_h_qbr">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_dpsu_h_qbl: GCCBuiltin<"__builtin_mips_dpsu_h_qbl">, +def int_mips_dpsu_h_qbl: ClangBuiltin<"__builtin_mips_dpsu_h_qbl">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_dpsu_h_qbr: GCCBuiltin<"__builtin_mips_dpsu_h_qbr">, +def int_mips_dpsu_h_qbr: ClangBuiltin<"__builtin_mips_dpsu_h_qbr">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_dpaq_s_w_ph: GCCBuiltin<"__builtin_mips_dpaq_s_w_ph">, +def int_mips_dpaq_s_w_ph: ClangBuiltin<"__builtin_mips_dpaq_s_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_dpsq_s_w_ph: GCCBuiltin<"__builtin_mips_dpsq_s_w_ph">, +def int_mips_dpsq_s_w_ph: ClangBuiltin<"__builtin_mips_dpsq_s_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_dpaq_sa_l_w: GCCBuiltin<"__builtin_mips_dpaq_sa_l_w">, +def int_mips_dpaq_sa_l_w: ClangBuiltin<"__builtin_mips_dpaq_sa_l_w">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_q31_ty, mips_q31_ty], []>; -def int_mips_dpsq_sa_l_w: GCCBuiltin<"__builtin_mips_dpsq_sa_l_w">, +def int_mips_dpsq_sa_l_w: ClangBuiltin<"__builtin_mips_dpsq_sa_l_w">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_q31_ty, mips_q31_ty], []>; //===----------------------------------------------------------------------===// // Comparison -def int_mips_cmpu_eq_qb: GCCBuiltin<"__builtin_mips_cmpu_eq_qb">, +def int_mips_cmpu_eq_qb: ClangBuiltin<"__builtin_mips_cmpu_eq_qb">, Intrinsic<[], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative]>; -def int_mips_cmpu_lt_qb: GCCBuiltin<"__builtin_mips_cmpu_lt_qb">, +def int_mips_cmpu_lt_qb: ClangBuiltin<"__builtin_mips_cmpu_lt_qb">, Intrinsic<[], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_cmpu_le_qb: GCCBuiltin<"__builtin_mips_cmpu_le_qb">, +def int_mips_cmpu_le_qb: ClangBuiltin<"__builtin_mips_cmpu_le_qb">, Intrinsic<[], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_cmpgu_eq_qb: GCCBuiltin<"__builtin_mips_cmpgu_eq_qb">, +def int_mips_cmpgu_eq_qb: ClangBuiltin<"__builtin_mips_cmpgu_eq_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative]>; -def int_mips_cmpgu_lt_qb: GCCBuiltin<"__builtin_mips_cmpgu_lt_qb">, +def int_mips_cmpgu_lt_qb: ClangBuiltin<"__builtin_mips_cmpgu_lt_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_cmpgu_le_qb: GCCBuiltin<"__builtin_mips_cmpgu_le_qb">, +def int_mips_cmpgu_le_qb: ClangBuiltin<"__builtin_mips_cmpgu_le_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_cmp_eq_ph: GCCBuiltin<"__builtin_mips_cmp_eq_ph">, +def int_mips_cmp_eq_ph: ClangBuiltin<"__builtin_mips_cmp_eq_ph">, Intrinsic<[], [mips_v2q15_ty, mips_v2q15_ty], [Commutative]>; -def int_mips_cmp_lt_ph: GCCBuiltin<"__builtin_mips_cmp_lt_ph">, +def int_mips_cmp_lt_ph: ClangBuiltin<"__builtin_mips_cmp_lt_ph">, Intrinsic<[], [mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_cmp_le_ph: GCCBuiltin<"__builtin_mips_cmp_le_ph">, +def int_mips_cmp_le_ph: ClangBuiltin<"__builtin_mips_cmp_le_ph">, Intrinsic<[], [mips_v2q15_ty, mips_v2q15_ty], []>; //===----------------------------------------------------------------------===// // Extracting -def int_mips_extr_s_h: GCCBuiltin<"__builtin_mips_extr_s_h">, +def int_mips_extr_s_h: ClangBuiltin<"__builtin_mips_extr_s_h">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_extr_w: GCCBuiltin<"__builtin_mips_extr_w">, +def int_mips_extr_w: ClangBuiltin<"__builtin_mips_extr_w">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_extr_rs_w: GCCBuiltin<"__builtin_mips_extr_rs_w">, +def int_mips_extr_rs_w: ClangBuiltin<"__builtin_mips_extr_rs_w">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_extr_r_w: GCCBuiltin<"__builtin_mips_extr_r_w">, +def int_mips_extr_r_w: ClangBuiltin<"__builtin_mips_extr_r_w">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_extp: GCCBuiltin<"__builtin_mips_extp">, +def int_mips_extp: ClangBuiltin<"__builtin_mips_extp">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_extpdp: GCCBuiltin<"__builtin_mips_extpdp">, +def int_mips_extpdp: ClangBuiltin<"__builtin_mips_extpdp">, Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_i32_ty], []>; //===----------------------------------------------------------------------===// // Misc -def int_mips_wrdsp: GCCBuiltin<"__builtin_mips_wrdsp">, +def int_mips_wrdsp: ClangBuiltin<"__builtin_mips_wrdsp">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>]>; -def int_mips_rddsp: GCCBuiltin<"__builtin_mips_rddsp">, +def int_mips_rddsp: ClangBuiltin<"__builtin_mips_rddsp">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<ArgIndex<0>>]>; -def int_mips_insv: GCCBuiltin<"__builtin_mips_insv">, +def int_mips_insv: ClangBuiltin<"__builtin_mips_insv">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>; -def int_mips_bitrev: GCCBuiltin<"__builtin_mips_bitrev">, +def int_mips_bitrev: ClangBuiltin<"__builtin_mips_bitrev">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_packrl_ph: GCCBuiltin<"__builtin_mips_packrl_ph">, +def int_mips_packrl_ph: ClangBuiltin<"__builtin_mips_packrl_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_repl_qb: GCCBuiltin<"__builtin_mips_repl_qb">, +def int_mips_repl_qb: ClangBuiltin<"__builtin_mips_repl_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_repl_ph: GCCBuiltin<"__builtin_mips_repl_ph">, +def int_mips_repl_ph: ClangBuiltin<"__builtin_mips_repl_ph">, Intrinsic<[mips_v2q15_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_pick_qb: GCCBuiltin<"__builtin_mips_pick_qb">, +def int_mips_pick_qb: ClangBuiltin<"__builtin_mips_pick_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrReadMem]>; -def int_mips_pick_ph: GCCBuiltin<"__builtin_mips_pick_ph">, +def int_mips_pick_ph: ClangBuiltin<"__builtin_mips_pick_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrReadMem]>; -def int_mips_mthlip: GCCBuiltin<"__builtin_mips_mthlip">, +def int_mips_mthlip: ClangBuiltin<"__builtin_mips_mthlip">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], []>; -def int_mips_bposge32: GCCBuiltin<"__builtin_mips_bposge32">, +def int_mips_bposge32: ClangBuiltin<"__builtin_mips_bposge32">, Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>; -def int_mips_lbux: GCCBuiltin<"__builtin_mips_lbux">, +def int_mips_lbux: ClangBuiltin<"__builtin_mips_lbux">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_lhx: GCCBuiltin<"__builtin_mips_lhx">, +def int_mips_lhx: ClangBuiltin<"__builtin_mips_lhx">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_lwx: GCCBuiltin<"__builtin_mips_lwx">, +def int_mips_lwx: ClangBuiltin<"__builtin_mips_lwx">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; //===----------------------------------------------------------------------===// // MIPS DSP Rev 2 -def int_mips_absq_s_qb: GCCBuiltin<"__builtin_mips_absq_s_qb">, +def int_mips_absq_s_qb: ClangBuiltin<"__builtin_mips_absq_s_qb">, Intrinsic<[mips_v4q7_ty], [mips_v4q7_ty], []>; -def int_mips_addqh_ph: GCCBuiltin<"__builtin_mips_addqh_ph">, +def int_mips_addqh_ph: ClangBuiltin<"__builtin_mips_addqh_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem, Commutative]>; -def int_mips_addqh_r_ph: GCCBuiltin<"__builtin_mips_addqh_r_ph">, +def int_mips_addqh_r_ph: ClangBuiltin<"__builtin_mips_addqh_r_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem, Commutative]>; -def int_mips_addqh_w: GCCBuiltin<"__builtin_mips_addqh_w">, +def int_mips_addqh_w: ClangBuiltin<"__builtin_mips_addqh_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [IntrNoMem, Commutative]>; -def int_mips_addqh_r_w: GCCBuiltin<"__builtin_mips_addqh_r_w">, +def int_mips_addqh_r_w: ClangBuiltin<"__builtin_mips_addqh_r_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [IntrNoMem, Commutative]>; -def int_mips_addu_ph: GCCBuiltin<"__builtin_mips_addu_ph">, +def int_mips_addu_ph: ClangBuiltin<"__builtin_mips_addu_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], [Commutative]>; -def int_mips_addu_s_ph: GCCBuiltin<"__builtin_mips_addu_s_ph">, +def int_mips_addu_s_ph: ClangBuiltin<"__builtin_mips_addu_s_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], [Commutative]>; -def int_mips_adduh_qb: GCCBuiltin<"__builtin_mips_adduh_qb">, +def int_mips_adduh_qb: ClangBuiltin<"__builtin_mips_adduh_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem, Commutative]>; -def int_mips_adduh_r_qb: GCCBuiltin<"__builtin_mips_adduh_r_qb">, +def int_mips_adduh_r_qb: ClangBuiltin<"__builtin_mips_adduh_r_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem, Commutative]>; -def int_mips_append: GCCBuiltin<"__builtin_mips_append">, +def int_mips_append: ClangBuiltin<"__builtin_mips_append">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_balign: GCCBuiltin<"__builtin_mips_balign">, +def int_mips_balign: ClangBuiltin<"__builtin_mips_balign">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_cmpgdu_eq_qb: GCCBuiltin<"__builtin_mips_cmpgdu_eq_qb">, +def int_mips_cmpgdu_eq_qb: ClangBuiltin<"__builtin_mips_cmpgdu_eq_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative]>; -def int_mips_cmpgdu_lt_qb: GCCBuiltin<"__builtin_mips_cmpgdu_lt_qb">, +def int_mips_cmpgdu_lt_qb: ClangBuiltin<"__builtin_mips_cmpgdu_lt_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_cmpgdu_le_qb: GCCBuiltin<"__builtin_mips_cmpgdu_le_qb">, +def int_mips_cmpgdu_le_qb: ClangBuiltin<"__builtin_mips_cmpgdu_le_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], []>; -def int_mips_dpa_w_ph: GCCBuiltin<"__builtin_mips_dpa_w_ph">, +def int_mips_dpa_w_ph: ClangBuiltin<"__builtin_mips_dpa_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v2i16_ty, llvm_v2i16_ty], [IntrNoMem]>; -def int_mips_dps_w_ph: GCCBuiltin<"__builtin_mips_dps_w_ph">, +def int_mips_dps_w_ph: ClangBuiltin<"__builtin_mips_dps_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v2i16_ty, llvm_v2i16_ty], [IntrNoMem]>; -def int_mips_dpaqx_s_w_ph: GCCBuiltin<"__builtin_mips_dpaqx_s_w_ph">, +def int_mips_dpaqx_s_w_ph: ClangBuiltin<"__builtin_mips_dpaqx_s_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_dpaqx_sa_w_ph: GCCBuiltin<"__builtin_mips_dpaqx_sa_w_ph">, +def int_mips_dpaqx_sa_w_ph: ClangBuiltin<"__builtin_mips_dpaqx_sa_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_dpax_w_ph: GCCBuiltin<"__builtin_mips_dpax_w_ph">, +def int_mips_dpax_w_ph: ClangBuiltin<"__builtin_mips_dpax_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v2i16_ty, llvm_v2i16_ty], [IntrNoMem]>; -def int_mips_dpsx_w_ph: GCCBuiltin<"__builtin_mips_dpsx_w_ph">, +def int_mips_dpsx_w_ph: ClangBuiltin<"__builtin_mips_dpsx_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v2i16_ty, llvm_v2i16_ty], [IntrNoMem]>; -def int_mips_dpsqx_s_w_ph: GCCBuiltin<"__builtin_mips_dpsqx_s_w_ph">, +def int_mips_dpsqx_s_w_ph: ClangBuiltin<"__builtin_mips_dpsqx_s_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_dpsqx_sa_w_ph: GCCBuiltin<"__builtin_mips_dpsqx_sa_w_ph">, +def int_mips_dpsqx_sa_w_ph: ClangBuiltin<"__builtin_mips_dpsqx_sa_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, mips_v2q15_ty, mips_v2q15_ty], []>; -def int_mips_mul_ph: GCCBuiltin<"__builtin_mips_mul_ph">, +def int_mips_mul_ph: ClangBuiltin<"__builtin_mips_mul_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], [Commutative]>; -def int_mips_mul_s_ph: GCCBuiltin<"__builtin_mips_mul_s_ph">, +def int_mips_mul_s_ph: ClangBuiltin<"__builtin_mips_mul_s_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], [Commutative]>; -def int_mips_mulq_rs_w: GCCBuiltin<"__builtin_mips_mulq_rs_w">, +def int_mips_mulq_rs_w: ClangBuiltin<"__builtin_mips_mulq_rs_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [Commutative]>; -def int_mips_mulq_s_ph: GCCBuiltin<"__builtin_mips_mulq_s_ph">, +def int_mips_mulq_s_ph: ClangBuiltin<"__builtin_mips_mulq_s_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [Commutative]>; -def int_mips_mulq_s_w: GCCBuiltin<"__builtin_mips_mulq_s_w">, +def int_mips_mulq_s_w: ClangBuiltin<"__builtin_mips_mulq_s_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [Commutative]>; -def int_mips_mulsa_w_ph: GCCBuiltin<"__builtin_mips_mulsa_w_ph">, +def int_mips_mulsa_w_ph: ClangBuiltin<"__builtin_mips_mulsa_w_ph">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_v2i16_ty, llvm_v2i16_ty], [IntrNoMem]>; -def int_mips_precr_qb_ph: GCCBuiltin<"__builtin_mips_precr_qb_ph">, +def int_mips_precr_qb_ph: ClangBuiltin<"__builtin_mips_precr_qb_ph">, Intrinsic<[llvm_v4i8_ty], [llvm_v2i16_ty, llvm_v2i16_ty], []>; -def int_mips_precr_sra_ph_w: GCCBuiltin<"__builtin_mips_precr_sra_ph_w">, +def int_mips_precr_sra_ph_w: ClangBuiltin<"__builtin_mips_precr_sra_ph_w">, Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_precr_sra_r_ph_w: GCCBuiltin<"__builtin_mips_precr_sra_r_ph_w">, +def int_mips_precr_sra_r_ph_w: ClangBuiltin<"__builtin_mips_precr_sra_r_ph_w">, Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_prepend: GCCBuiltin<"__builtin_mips_prepend">, +def int_mips_prepend: ClangBuiltin<"__builtin_mips_prepend">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_shra_qb: GCCBuiltin<"__builtin_mips_shra_qb">, +def int_mips_shra_qb: ClangBuiltin<"__builtin_mips_shra_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shra_r_qb: GCCBuiltin<"__builtin_mips_shra_r_qb">, +def int_mips_shra_r_qb: ClangBuiltin<"__builtin_mips_shra_r_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_shrl_ph: GCCBuiltin<"__builtin_mips_shrl_ph">, +def int_mips_shrl_ph: ClangBuiltin<"__builtin_mips_shrl_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_subqh_ph: GCCBuiltin<"__builtin_mips_subqh_ph">, +def int_mips_subqh_ph: ClangBuiltin<"__builtin_mips_subqh_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_subqh_r_ph: GCCBuiltin<"__builtin_mips_subqh_r_ph">, +def int_mips_subqh_r_ph: ClangBuiltin<"__builtin_mips_subqh_r_ph">, Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrNoMem]>; -def int_mips_subqh_w: GCCBuiltin<"__builtin_mips_subqh_w">, +def int_mips_subqh_w: ClangBuiltin<"__builtin_mips_subqh_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [IntrNoMem]>; -def int_mips_subqh_r_w: GCCBuiltin<"__builtin_mips_subqh_r_w">, +def int_mips_subqh_r_w: ClangBuiltin<"__builtin_mips_subqh_r_w">, Intrinsic<[mips_q31_ty], [mips_q31_ty, mips_q31_ty], [IntrNoMem]>; -def int_mips_subu_ph: GCCBuiltin<"__builtin_mips_subu_ph">, +def int_mips_subu_ph: ClangBuiltin<"__builtin_mips_subu_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], []>; -def int_mips_subu_s_ph: GCCBuiltin<"__builtin_mips_subu_s_ph">, +def int_mips_subu_s_ph: ClangBuiltin<"__builtin_mips_subu_s_ph">, Intrinsic<[llvm_v2i16_ty], [llvm_v2i16_ty, llvm_v2i16_ty], []>; -def int_mips_subuh_qb: GCCBuiltin<"__builtin_mips_subuh_qb">, +def int_mips_subuh_qb: ClangBuiltin<"__builtin_mips_subuh_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; -def int_mips_subuh_r_qb: GCCBuiltin<"__builtin_mips_subuh_r_qb">, +def int_mips_subuh_r_qb: ClangBuiltin<"__builtin_mips_subuh_r_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrNoMem]>; //===----------------------------------------------------------------------===// @@ -396,1389 +396,1389 @@ def int_mips_subuh_r_qb: GCCBuiltin<"__builtin_mips_subuh_r_qb">, //===----------------------------------------------------------------------===// // Addition/subtraction -def int_mips_add_a_b : GCCBuiltin<"__builtin_msa_add_a_b">, +def int_mips_add_a_b : ClangBuiltin<"__builtin_msa_add_a_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_add_a_h : GCCBuiltin<"__builtin_msa_add_a_h">, +def int_mips_add_a_h : ClangBuiltin<"__builtin_msa_add_a_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_add_a_w : GCCBuiltin<"__builtin_msa_add_a_w">, +def int_mips_add_a_w : ClangBuiltin<"__builtin_msa_add_a_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_add_a_d : GCCBuiltin<"__builtin_msa_add_a_d">, +def int_mips_add_a_d : ClangBuiltin<"__builtin_msa_add_a_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_a_b : GCCBuiltin<"__builtin_msa_adds_a_b">, +def int_mips_adds_a_b : ClangBuiltin<"__builtin_msa_adds_a_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_a_h : GCCBuiltin<"__builtin_msa_adds_a_h">, +def int_mips_adds_a_h : ClangBuiltin<"__builtin_msa_adds_a_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_a_w : GCCBuiltin<"__builtin_msa_adds_a_w">, +def int_mips_adds_a_w : ClangBuiltin<"__builtin_msa_adds_a_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_a_d : GCCBuiltin<"__builtin_msa_adds_a_d">, +def int_mips_adds_a_d : ClangBuiltin<"__builtin_msa_adds_a_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_s_b : GCCBuiltin<"__builtin_msa_adds_s_b">, +def int_mips_adds_s_b : ClangBuiltin<"__builtin_msa_adds_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_s_h : GCCBuiltin<"__builtin_msa_adds_s_h">, +def int_mips_adds_s_h : ClangBuiltin<"__builtin_msa_adds_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_s_w : GCCBuiltin<"__builtin_msa_adds_s_w">, +def int_mips_adds_s_w : ClangBuiltin<"__builtin_msa_adds_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_s_d : GCCBuiltin<"__builtin_msa_adds_s_d">, +def int_mips_adds_s_d : ClangBuiltin<"__builtin_msa_adds_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_u_b : GCCBuiltin<"__builtin_msa_adds_u_b">, +def int_mips_adds_u_b : ClangBuiltin<"__builtin_msa_adds_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_u_h : GCCBuiltin<"__builtin_msa_adds_u_h">, +def int_mips_adds_u_h : ClangBuiltin<"__builtin_msa_adds_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_u_w : GCCBuiltin<"__builtin_msa_adds_u_w">, +def int_mips_adds_u_w : ClangBuiltin<"__builtin_msa_adds_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_adds_u_d : GCCBuiltin<"__builtin_msa_adds_u_d">, +def int_mips_adds_u_d : ClangBuiltin<"__builtin_msa_adds_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_addv_b : GCCBuiltin<"__builtin_msa_addv_b">, +def int_mips_addv_b : ClangBuiltin<"__builtin_msa_addv_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_addv_h : GCCBuiltin<"__builtin_msa_addv_h">, +def int_mips_addv_h : ClangBuiltin<"__builtin_msa_addv_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_addv_w : GCCBuiltin<"__builtin_msa_addv_w">, +def int_mips_addv_w : ClangBuiltin<"__builtin_msa_addv_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_addv_d : GCCBuiltin<"__builtin_msa_addv_d">, +def int_mips_addv_d : ClangBuiltin<"__builtin_msa_addv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_addvi_b : GCCBuiltin<"__builtin_msa_addvi_b">, +def int_mips_addvi_b : ClangBuiltin<"__builtin_msa_addvi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [Commutative, IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_addvi_h : GCCBuiltin<"__builtin_msa_addvi_h">, +def int_mips_addvi_h : ClangBuiltin<"__builtin_msa_addvi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [Commutative, IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_addvi_w : GCCBuiltin<"__builtin_msa_addvi_w">, +def int_mips_addvi_w : ClangBuiltin<"__builtin_msa_addvi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [Commutative, IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_addvi_d : GCCBuiltin<"__builtin_msa_addvi_d">, +def int_mips_addvi_d : ClangBuiltin<"__builtin_msa_addvi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [Commutative, IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_and_v : GCCBuiltin<"__builtin_msa_and_v">, +def int_mips_and_v : ClangBuiltin<"__builtin_msa_and_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_andi_b : GCCBuiltin<"__builtin_msa_andi_b">, +def int_mips_andi_b : ClangBuiltin<"__builtin_msa_andi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_asub_s_b : GCCBuiltin<"__builtin_msa_asub_s_b">, +def int_mips_asub_s_b : ClangBuiltin<"__builtin_msa_asub_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_asub_s_h : GCCBuiltin<"__builtin_msa_asub_s_h">, +def int_mips_asub_s_h : ClangBuiltin<"__builtin_msa_asub_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_asub_s_w : GCCBuiltin<"__builtin_msa_asub_s_w">, +def int_mips_asub_s_w : ClangBuiltin<"__builtin_msa_asub_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_asub_s_d : GCCBuiltin<"__builtin_msa_asub_s_d">, +def int_mips_asub_s_d : ClangBuiltin<"__builtin_msa_asub_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_asub_u_b : GCCBuiltin<"__builtin_msa_asub_u_b">, +def int_mips_asub_u_b : ClangBuiltin<"__builtin_msa_asub_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_asub_u_h : GCCBuiltin<"__builtin_msa_asub_u_h">, +def int_mips_asub_u_h : ClangBuiltin<"__builtin_msa_asub_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_asub_u_w : GCCBuiltin<"__builtin_msa_asub_u_w">, +def int_mips_asub_u_w : ClangBuiltin<"__builtin_msa_asub_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_asub_u_d : GCCBuiltin<"__builtin_msa_asub_u_d">, +def int_mips_asub_u_d : ClangBuiltin<"__builtin_msa_asub_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ave_s_b : GCCBuiltin<"__builtin_msa_ave_s_b">, +def int_mips_ave_s_b : ClangBuiltin<"__builtin_msa_ave_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_s_h : GCCBuiltin<"__builtin_msa_ave_s_h">, +def int_mips_ave_s_h : ClangBuiltin<"__builtin_msa_ave_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_s_w : GCCBuiltin<"__builtin_msa_ave_s_w">, +def int_mips_ave_s_w : ClangBuiltin<"__builtin_msa_ave_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_s_d : GCCBuiltin<"__builtin_msa_ave_s_d">, +def int_mips_ave_s_d : ClangBuiltin<"__builtin_msa_ave_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_u_b : GCCBuiltin<"__builtin_msa_ave_u_b">, +def int_mips_ave_u_b : ClangBuiltin<"__builtin_msa_ave_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_u_h : GCCBuiltin<"__builtin_msa_ave_u_h">, +def int_mips_ave_u_h : ClangBuiltin<"__builtin_msa_ave_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_u_w : GCCBuiltin<"__builtin_msa_ave_u_w">, +def int_mips_ave_u_w : ClangBuiltin<"__builtin_msa_ave_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_ave_u_d : GCCBuiltin<"__builtin_msa_ave_u_d">, +def int_mips_ave_u_d : ClangBuiltin<"__builtin_msa_ave_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_s_b : GCCBuiltin<"__builtin_msa_aver_s_b">, +def int_mips_aver_s_b : ClangBuiltin<"__builtin_msa_aver_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_s_h : GCCBuiltin<"__builtin_msa_aver_s_h">, +def int_mips_aver_s_h : ClangBuiltin<"__builtin_msa_aver_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_s_w : GCCBuiltin<"__builtin_msa_aver_s_w">, +def int_mips_aver_s_w : ClangBuiltin<"__builtin_msa_aver_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_s_d : GCCBuiltin<"__builtin_msa_aver_s_d">, +def int_mips_aver_s_d : ClangBuiltin<"__builtin_msa_aver_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_u_b : GCCBuiltin<"__builtin_msa_aver_u_b">, +def int_mips_aver_u_b : ClangBuiltin<"__builtin_msa_aver_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_u_h : GCCBuiltin<"__builtin_msa_aver_u_h">, +def int_mips_aver_u_h : ClangBuiltin<"__builtin_msa_aver_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_u_w : GCCBuiltin<"__builtin_msa_aver_u_w">, +def int_mips_aver_u_w : ClangBuiltin<"__builtin_msa_aver_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [Commutative, IntrNoMem]>; -def int_mips_aver_u_d : GCCBuiltin<"__builtin_msa_aver_u_d">, +def int_mips_aver_u_d : ClangBuiltin<"__builtin_msa_aver_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [Commutative, IntrNoMem]>; -def int_mips_bclr_b : GCCBuiltin<"__builtin_msa_bclr_b">, +def int_mips_bclr_b : ClangBuiltin<"__builtin_msa_bclr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bclr_h : GCCBuiltin<"__builtin_msa_bclr_h">, +def int_mips_bclr_h : ClangBuiltin<"__builtin_msa_bclr_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_bclr_w : GCCBuiltin<"__builtin_msa_bclr_w">, +def int_mips_bclr_w : ClangBuiltin<"__builtin_msa_bclr_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_bclr_d : GCCBuiltin<"__builtin_msa_bclr_d">, +def int_mips_bclr_d : ClangBuiltin<"__builtin_msa_bclr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_bclri_b : GCCBuiltin<"__builtin_msa_bclri_b">, +def int_mips_bclri_b : ClangBuiltin<"__builtin_msa_bclri_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bclri_h : GCCBuiltin<"__builtin_msa_bclri_h">, +def int_mips_bclri_h : ClangBuiltin<"__builtin_msa_bclri_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bclri_w : GCCBuiltin<"__builtin_msa_bclri_w">, +def int_mips_bclri_w : ClangBuiltin<"__builtin_msa_bclri_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bclri_d : GCCBuiltin<"__builtin_msa_bclri_d">, +def int_mips_bclri_d : ClangBuiltin<"__builtin_msa_bclri_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_binsl_b : GCCBuiltin<"__builtin_msa_binsl_b">, +def int_mips_binsl_b : ClangBuiltin<"__builtin_msa_binsl_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_binsl_h : GCCBuiltin<"__builtin_msa_binsl_h">, +def int_mips_binsl_h : ClangBuiltin<"__builtin_msa_binsl_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_binsl_w : GCCBuiltin<"__builtin_msa_binsl_w">, +def int_mips_binsl_w : ClangBuiltin<"__builtin_msa_binsl_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_binsl_d : GCCBuiltin<"__builtin_msa_binsl_d">, +def int_mips_binsl_d : ClangBuiltin<"__builtin_msa_binsl_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_binsli_b : GCCBuiltin<"__builtin_msa_binsli_b">, +def int_mips_binsli_b : ClangBuiltin<"__builtin_msa_binsli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsli_h : GCCBuiltin<"__builtin_msa_binsli_h">, +def int_mips_binsli_h : ClangBuiltin<"__builtin_msa_binsli_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsli_w : GCCBuiltin<"__builtin_msa_binsli_w">, +def int_mips_binsli_w : ClangBuiltin<"__builtin_msa_binsli_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsli_d : GCCBuiltin<"__builtin_msa_binsli_d">, +def int_mips_binsli_d : ClangBuiltin<"__builtin_msa_binsli_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsr_b : GCCBuiltin<"__builtin_msa_binsr_b">, +def int_mips_binsr_b : ClangBuiltin<"__builtin_msa_binsr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_binsr_h : GCCBuiltin<"__builtin_msa_binsr_h">, +def int_mips_binsr_h : ClangBuiltin<"__builtin_msa_binsr_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_binsr_w : GCCBuiltin<"__builtin_msa_binsr_w">, +def int_mips_binsr_w : ClangBuiltin<"__builtin_msa_binsr_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_binsr_d : GCCBuiltin<"__builtin_msa_binsr_d">, +def int_mips_binsr_d : ClangBuiltin<"__builtin_msa_binsr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_binsri_b : GCCBuiltin<"__builtin_msa_binsri_b">, +def int_mips_binsri_b : ClangBuiltin<"__builtin_msa_binsri_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsri_h : GCCBuiltin<"__builtin_msa_binsri_h">, +def int_mips_binsri_h : ClangBuiltin<"__builtin_msa_binsri_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsri_w : GCCBuiltin<"__builtin_msa_binsri_w">, +def int_mips_binsri_w : ClangBuiltin<"__builtin_msa_binsri_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_binsri_d : GCCBuiltin<"__builtin_msa_binsri_d">, +def int_mips_binsri_d : ClangBuiltin<"__builtin_msa_binsri_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_bmnz_v : GCCBuiltin<"__builtin_msa_bmnz_v">, +def int_mips_bmnz_v : ClangBuiltin<"__builtin_msa_bmnz_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bmnzi_b : GCCBuiltin<"__builtin_msa_bmnzi_b">, +def int_mips_bmnzi_b : ClangBuiltin<"__builtin_msa_bmnzi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_bmz_v : GCCBuiltin<"__builtin_msa_bmz_v">, +def int_mips_bmz_v : ClangBuiltin<"__builtin_msa_bmz_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bmzi_b : GCCBuiltin<"__builtin_msa_bmzi_b">, +def int_mips_bmzi_b : ClangBuiltin<"__builtin_msa_bmzi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_bneg_b : GCCBuiltin<"__builtin_msa_bneg_b">, +def int_mips_bneg_b : ClangBuiltin<"__builtin_msa_bneg_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bneg_h : GCCBuiltin<"__builtin_msa_bneg_h">, +def int_mips_bneg_h : ClangBuiltin<"__builtin_msa_bneg_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_bneg_w : GCCBuiltin<"__builtin_msa_bneg_w">, +def int_mips_bneg_w : ClangBuiltin<"__builtin_msa_bneg_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_bneg_d : GCCBuiltin<"__builtin_msa_bneg_d">, +def int_mips_bneg_d : ClangBuiltin<"__builtin_msa_bneg_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_bnegi_b : GCCBuiltin<"__builtin_msa_bnegi_b">, +def int_mips_bnegi_b : ClangBuiltin<"__builtin_msa_bnegi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bnegi_h : GCCBuiltin<"__builtin_msa_bnegi_h">, +def int_mips_bnegi_h : ClangBuiltin<"__builtin_msa_bnegi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bnegi_w : GCCBuiltin<"__builtin_msa_bnegi_w">, +def int_mips_bnegi_w : ClangBuiltin<"__builtin_msa_bnegi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bnegi_d : GCCBuiltin<"__builtin_msa_bnegi_d">, +def int_mips_bnegi_d : ClangBuiltin<"__builtin_msa_bnegi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bnz_b : GCCBuiltin<"__builtin_msa_bnz_b">, +def int_mips_bnz_b : ClangBuiltin<"__builtin_msa_bnz_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bnz_h : GCCBuiltin<"__builtin_msa_bnz_h">, +def int_mips_bnz_h : ClangBuiltin<"__builtin_msa_bnz_h">, Intrinsic<[llvm_i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_bnz_w : GCCBuiltin<"__builtin_msa_bnz_w">, +def int_mips_bnz_w : ClangBuiltin<"__builtin_msa_bnz_w">, Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_bnz_d : GCCBuiltin<"__builtin_msa_bnz_d">, +def int_mips_bnz_d : ClangBuiltin<"__builtin_msa_bnz_d">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_bnz_v : GCCBuiltin<"__builtin_msa_bnz_v">, +def int_mips_bnz_v : ClangBuiltin<"__builtin_msa_bnz_v">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bsel_v : GCCBuiltin<"__builtin_msa_bsel_v">, +def int_mips_bsel_v : ClangBuiltin<"__builtin_msa_bsel_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bseli_b : GCCBuiltin<"__builtin_msa_bseli_b">, +def int_mips_bseli_b : ClangBuiltin<"__builtin_msa_bseli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_bset_b : GCCBuiltin<"__builtin_msa_bset_b">, +def int_mips_bset_b : ClangBuiltin<"__builtin_msa_bset_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bset_h : GCCBuiltin<"__builtin_msa_bset_h">, +def int_mips_bset_h : ClangBuiltin<"__builtin_msa_bset_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_bset_w : GCCBuiltin<"__builtin_msa_bset_w">, +def int_mips_bset_w : ClangBuiltin<"__builtin_msa_bset_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_bset_d : GCCBuiltin<"__builtin_msa_bset_d">, +def int_mips_bset_d : ClangBuiltin<"__builtin_msa_bset_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_bseti_b : GCCBuiltin<"__builtin_msa_bseti_b">, +def int_mips_bseti_b : ClangBuiltin<"__builtin_msa_bseti_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bseti_h : GCCBuiltin<"__builtin_msa_bseti_h">, +def int_mips_bseti_h : ClangBuiltin<"__builtin_msa_bseti_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bseti_w : GCCBuiltin<"__builtin_msa_bseti_w">, +def int_mips_bseti_w : ClangBuiltin<"__builtin_msa_bseti_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bseti_d : GCCBuiltin<"__builtin_msa_bseti_d">, +def int_mips_bseti_d : ClangBuiltin<"__builtin_msa_bseti_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_bz_b : GCCBuiltin<"__builtin_msa_bz_b">, +def int_mips_bz_b : ClangBuiltin<"__builtin_msa_bz_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_bz_h : GCCBuiltin<"__builtin_msa_bz_h">, +def int_mips_bz_h : ClangBuiltin<"__builtin_msa_bz_h">, Intrinsic<[llvm_i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_bz_w : GCCBuiltin<"__builtin_msa_bz_w">, +def int_mips_bz_w : ClangBuiltin<"__builtin_msa_bz_w">, Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_bz_d : GCCBuiltin<"__builtin_msa_bz_d">, +def int_mips_bz_d : ClangBuiltin<"__builtin_msa_bz_d">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_bz_v : GCCBuiltin<"__builtin_msa_bz_v">, +def int_mips_bz_v : ClangBuiltin<"__builtin_msa_bz_v">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ceq_b : GCCBuiltin<"__builtin_msa_ceq_b">, +def int_mips_ceq_b : ClangBuiltin<"__builtin_msa_ceq_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ceq_h : GCCBuiltin<"__builtin_msa_ceq_h">, +def int_mips_ceq_h : ClangBuiltin<"__builtin_msa_ceq_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ceq_w : GCCBuiltin<"__builtin_msa_ceq_w">, +def int_mips_ceq_w : ClangBuiltin<"__builtin_msa_ceq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ceq_d : GCCBuiltin<"__builtin_msa_ceq_d">, +def int_mips_ceq_d : ClangBuiltin<"__builtin_msa_ceq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ceqi_b : GCCBuiltin<"__builtin_msa_ceqi_b">, +def int_mips_ceqi_b : ClangBuiltin<"__builtin_msa_ceqi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_ceqi_h : GCCBuiltin<"__builtin_msa_ceqi_h">, +def int_mips_ceqi_h : ClangBuiltin<"__builtin_msa_ceqi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_ceqi_w : GCCBuiltin<"__builtin_msa_ceqi_w">, +def int_mips_ceqi_w : ClangBuiltin<"__builtin_msa_ceqi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_ceqi_d : GCCBuiltin<"__builtin_msa_ceqi_d">, +def int_mips_ceqi_d : ClangBuiltin<"__builtin_msa_ceqi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_cfcmsa : GCCBuiltin<"__builtin_msa_cfcmsa">, +def int_mips_cfcmsa : ClangBuiltin<"__builtin_msa_cfcmsa">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; -def int_mips_cle_s_b : GCCBuiltin<"__builtin_msa_cle_s_b">, +def int_mips_cle_s_b : ClangBuiltin<"__builtin_msa_cle_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_cle_s_h : GCCBuiltin<"__builtin_msa_cle_s_h">, +def int_mips_cle_s_h : ClangBuiltin<"__builtin_msa_cle_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_cle_s_w : GCCBuiltin<"__builtin_msa_cle_s_w">, +def int_mips_cle_s_w : ClangBuiltin<"__builtin_msa_cle_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_cle_s_d : GCCBuiltin<"__builtin_msa_cle_s_d">, +def int_mips_cle_s_d : ClangBuiltin<"__builtin_msa_cle_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_cle_u_b : GCCBuiltin<"__builtin_msa_cle_u_b">, +def int_mips_cle_u_b : ClangBuiltin<"__builtin_msa_cle_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_cle_u_h : GCCBuiltin<"__builtin_msa_cle_u_h">, +def int_mips_cle_u_h : ClangBuiltin<"__builtin_msa_cle_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_cle_u_w : GCCBuiltin<"__builtin_msa_cle_u_w">, +def int_mips_cle_u_w : ClangBuiltin<"__builtin_msa_cle_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_cle_u_d : GCCBuiltin<"__builtin_msa_cle_u_d">, +def int_mips_cle_u_d : ClangBuiltin<"__builtin_msa_cle_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_clei_s_b : GCCBuiltin<"__builtin_msa_clei_s_b">, +def int_mips_clei_s_b : ClangBuiltin<"__builtin_msa_clei_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_s_h : GCCBuiltin<"__builtin_msa_clei_s_h">, +def int_mips_clei_s_h : ClangBuiltin<"__builtin_msa_clei_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_s_w : GCCBuiltin<"__builtin_msa_clei_s_w">, +def int_mips_clei_s_w : ClangBuiltin<"__builtin_msa_clei_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_s_d : GCCBuiltin<"__builtin_msa_clei_s_d">, +def int_mips_clei_s_d : ClangBuiltin<"__builtin_msa_clei_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_u_b : GCCBuiltin<"__builtin_msa_clei_u_b">, +def int_mips_clei_u_b : ClangBuiltin<"__builtin_msa_clei_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_u_h : GCCBuiltin<"__builtin_msa_clei_u_h">, +def int_mips_clei_u_h : ClangBuiltin<"__builtin_msa_clei_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_u_w : GCCBuiltin<"__builtin_msa_clei_u_w">, +def int_mips_clei_u_w : ClangBuiltin<"__builtin_msa_clei_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clei_u_d : GCCBuiltin<"__builtin_msa_clei_u_d">, +def int_mips_clei_u_d : ClangBuiltin<"__builtin_msa_clei_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clt_s_b : GCCBuiltin<"__builtin_msa_clt_s_b">, +def int_mips_clt_s_b : ClangBuiltin<"__builtin_msa_clt_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_clt_s_h : GCCBuiltin<"__builtin_msa_clt_s_h">, +def int_mips_clt_s_h : ClangBuiltin<"__builtin_msa_clt_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_clt_s_w : GCCBuiltin<"__builtin_msa_clt_s_w">, +def int_mips_clt_s_w : ClangBuiltin<"__builtin_msa_clt_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_clt_s_d : GCCBuiltin<"__builtin_msa_clt_s_d">, +def int_mips_clt_s_d : ClangBuiltin<"__builtin_msa_clt_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_clt_u_b : GCCBuiltin<"__builtin_msa_clt_u_b">, +def int_mips_clt_u_b : ClangBuiltin<"__builtin_msa_clt_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_clt_u_h : GCCBuiltin<"__builtin_msa_clt_u_h">, +def int_mips_clt_u_h : ClangBuiltin<"__builtin_msa_clt_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_clt_u_w : GCCBuiltin<"__builtin_msa_clt_u_w">, +def int_mips_clt_u_w : ClangBuiltin<"__builtin_msa_clt_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_clt_u_d : GCCBuiltin<"__builtin_msa_clt_u_d">, +def int_mips_clt_u_d : ClangBuiltin<"__builtin_msa_clt_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_clti_s_b : GCCBuiltin<"__builtin_msa_clti_s_b">, +def int_mips_clti_s_b : ClangBuiltin<"__builtin_msa_clti_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_s_h : GCCBuiltin<"__builtin_msa_clti_s_h">, +def int_mips_clti_s_h : ClangBuiltin<"__builtin_msa_clti_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_s_w : GCCBuiltin<"__builtin_msa_clti_s_w">, +def int_mips_clti_s_w : ClangBuiltin<"__builtin_msa_clti_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_s_d : GCCBuiltin<"__builtin_msa_clti_s_d">, +def int_mips_clti_s_d : ClangBuiltin<"__builtin_msa_clti_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_u_b : GCCBuiltin<"__builtin_msa_clti_u_b">, +def int_mips_clti_u_b : ClangBuiltin<"__builtin_msa_clti_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_u_h : GCCBuiltin<"__builtin_msa_clti_u_h">, +def int_mips_clti_u_h : ClangBuiltin<"__builtin_msa_clti_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_u_w : GCCBuiltin<"__builtin_msa_clti_u_w">, +def int_mips_clti_u_w : ClangBuiltin<"__builtin_msa_clti_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_clti_u_d : GCCBuiltin<"__builtin_msa_clti_u_d">, +def int_mips_clti_u_d : ClangBuiltin<"__builtin_msa_clti_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_copy_s_b : GCCBuiltin<"__builtin_msa_copy_s_b">, +def int_mips_copy_s_b : ClangBuiltin<"__builtin_msa_copy_s_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_s_h : GCCBuiltin<"__builtin_msa_copy_s_h">, +def int_mips_copy_s_h : ClangBuiltin<"__builtin_msa_copy_s_h">, Intrinsic<[llvm_i32_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_s_w : GCCBuiltin<"__builtin_msa_copy_s_w">, +def int_mips_copy_s_w : ClangBuiltin<"__builtin_msa_copy_s_w">, Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_s_d : GCCBuiltin<"__builtin_msa_copy_s_d">, +def int_mips_copy_s_d : ClangBuiltin<"__builtin_msa_copy_s_d">, Intrinsic<[llvm_i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_u_b : GCCBuiltin<"__builtin_msa_copy_u_b">, +def int_mips_copy_u_b : ClangBuiltin<"__builtin_msa_copy_u_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_u_h : GCCBuiltin<"__builtin_msa_copy_u_h">, +def int_mips_copy_u_h : ClangBuiltin<"__builtin_msa_copy_u_h">, Intrinsic<[llvm_i32_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_u_w : GCCBuiltin<"__builtin_msa_copy_u_w">, +def int_mips_copy_u_w : ClangBuiltin<"__builtin_msa_copy_u_w">, Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_copy_u_d : GCCBuiltin<"__builtin_msa_copy_u_d">, +def int_mips_copy_u_d : ClangBuiltin<"__builtin_msa_copy_u_d">, Intrinsic<[llvm_i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_ctcmsa : GCCBuiltin<"__builtin_msa_ctcmsa">, +def int_mips_ctcmsa : ClangBuiltin<"__builtin_msa_ctcmsa">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; -def int_mips_div_s_b : GCCBuiltin<"__builtin_msa_div_s_b">, +def int_mips_div_s_b : ClangBuiltin<"__builtin_msa_div_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_div_s_h : GCCBuiltin<"__builtin_msa_div_s_h">, +def int_mips_div_s_h : ClangBuiltin<"__builtin_msa_div_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_div_s_w : GCCBuiltin<"__builtin_msa_div_s_w">, +def int_mips_div_s_w : ClangBuiltin<"__builtin_msa_div_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_div_s_d : GCCBuiltin<"__builtin_msa_div_s_d">, +def int_mips_div_s_d : ClangBuiltin<"__builtin_msa_div_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_div_u_b : GCCBuiltin<"__builtin_msa_div_u_b">, +def int_mips_div_u_b : ClangBuiltin<"__builtin_msa_div_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_div_u_h : GCCBuiltin<"__builtin_msa_div_u_h">, +def int_mips_div_u_h : ClangBuiltin<"__builtin_msa_div_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_div_u_w : GCCBuiltin<"__builtin_msa_div_u_w">, +def int_mips_div_u_w : ClangBuiltin<"__builtin_msa_div_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_div_u_d : GCCBuiltin<"__builtin_msa_div_u_d">, +def int_mips_div_u_d : ClangBuiltin<"__builtin_msa_div_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; // This instruction is part of the MSA spec but it does not share the // __builtin_msa prefix because it operates on GP registers. -def int_mips_dlsa : GCCBuiltin<"__builtin_mips_dlsa">, +def int_mips_dlsa : ClangBuiltin<"__builtin_mips_dlsa">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_dotp_s_h : GCCBuiltin<"__builtin_msa_dotp_s_h">, +def int_mips_dotp_s_h : ClangBuiltin<"__builtin_msa_dotp_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dotp_s_w : GCCBuiltin<"__builtin_msa_dotp_s_w">, +def int_mips_dotp_s_w : ClangBuiltin<"__builtin_msa_dotp_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dotp_s_d : GCCBuiltin<"__builtin_msa_dotp_s_d">, +def int_mips_dotp_s_d : ClangBuiltin<"__builtin_msa_dotp_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_dotp_u_h : GCCBuiltin<"__builtin_msa_dotp_u_h">, +def int_mips_dotp_u_h : ClangBuiltin<"__builtin_msa_dotp_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dotp_u_w : GCCBuiltin<"__builtin_msa_dotp_u_w">, +def int_mips_dotp_u_w : ClangBuiltin<"__builtin_msa_dotp_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dotp_u_d : GCCBuiltin<"__builtin_msa_dotp_u_d">, +def int_mips_dotp_u_d : ClangBuiltin<"__builtin_msa_dotp_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_dpadd_s_h : GCCBuiltin<"__builtin_msa_dpadd_s_h">, +def int_mips_dpadd_s_h : ClangBuiltin<"__builtin_msa_dpadd_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dpadd_s_w : GCCBuiltin<"__builtin_msa_dpadd_s_w">, +def int_mips_dpadd_s_w : ClangBuiltin<"__builtin_msa_dpadd_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dpadd_s_d : GCCBuiltin<"__builtin_msa_dpadd_s_d">, +def int_mips_dpadd_s_d : ClangBuiltin<"__builtin_msa_dpadd_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_dpadd_u_h : GCCBuiltin<"__builtin_msa_dpadd_u_h">, +def int_mips_dpadd_u_h : ClangBuiltin<"__builtin_msa_dpadd_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dpadd_u_w : GCCBuiltin<"__builtin_msa_dpadd_u_w">, +def int_mips_dpadd_u_w : ClangBuiltin<"__builtin_msa_dpadd_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dpadd_u_d : GCCBuiltin<"__builtin_msa_dpadd_u_d">, +def int_mips_dpadd_u_d : ClangBuiltin<"__builtin_msa_dpadd_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_dpsub_s_h : GCCBuiltin<"__builtin_msa_dpsub_s_h">, +def int_mips_dpsub_s_h : ClangBuiltin<"__builtin_msa_dpsub_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dpsub_s_w : GCCBuiltin<"__builtin_msa_dpsub_s_w">, +def int_mips_dpsub_s_w : ClangBuiltin<"__builtin_msa_dpsub_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dpsub_s_d : GCCBuiltin<"__builtin_msa_dpsub_s_d">, +def int_mips_dpsub_s_d : ClangBuiltin<"__builtin_msa_dpsub_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_dpsub_u_h : GCCBuiltin<"__builtin_msa_dpsub_u_h">, +def int_mips_dpsub_u_h : ClangBuiltin<"__builtin_msa_dpsub_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_dpsub_u_w : GCCBuiltin<"__builtin_msa_dpsub_u_w">, +def int_mips_dpsub_u_w : ClangBuiltin<"__builtin_msa_dpsub_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_dpsub_u_d : GCCBuiltin<"__builtin_msa_dpsub_u_d">, +def int_mips_dpsub_u_d : ClangBuiltin<"__builtin_msa_dpsub_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_fadd_w : GCCBuiltin<"__builtin_msa_fadd_w">, +def int_mips_fadd_w : ClangBuiltin<"__builtin_msa_fadd_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fadd_d : GCCBuiltin<"__builtin_msa_fadd_d">, +def int_mips_fadd_d : ClangBuiltin<"__builtin_msa_fadd_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcaf_w : GCCBuiltin<"__builtin_msa_fcaf_w">, +def int_mips_fcaf_w : ClangBuiltin<"__builtin_msa_fcaf_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcaf_d : GCCBuiltin<"__builtin_msa_fcaf_d">, +def int_mips_fcaf_d : ClangBuiltin<"__builtin_msa_fcaf_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fceq_w : GCCBuiltin<"__builtin_msa_fceq_w">, +def int_mips_fceq_w : ClangBuiltin<"__builtin_msa_fceq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fceq_d : GCCBuiltin<"__builtin_msa_fceq_d">, +def int_mips_fceq_d : ClangBuiltin<"__builtin_msa_fceq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcle_w : GCCBuiltin<"__builtin_msa_fcle_w">, +def int_mips_fcle_w : ClangBuiltin<"__builtin_msa_fcle_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcle_d : GCCBuiltin<"__builtin_msa_fcle_d">, +def int_mips_fcle_d : ClangBuiltin<"__builtin_msa_fcle_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fclt_w : GCCBuiltin<"__builtin_msa_fclt_w">, +def int_mips_fclt_w : ClangBuiltin<"__builtin_msa_fclt_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fclt_d : GCCBuiltin<"__builtin_msa_fclt_d">, +def int_mips_fclt_d : ClangBuiltin<"__builtin_msa_fclt_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fclass_w : GCCBuiltin<"__builtin_msa_fclass_w">, +def int_mips_fclass_w : ClangBuiltin<"__builtin_msa_fclass_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fclass_d : GCCBuiltin<"__builtin_msa_fclass_d">, +def int_mips_fclass_d : ClangBuiltin<"__builtin_msa_fclass_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcne_w : GCCBuiltin<"__builtin_msa_fcne_w">, +def int_mips_fcne_w : ClangBuiltin<"__builtin_msa_fcne_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcne_d : GCCBuiltin<"__builtin_msa_fcne_d">, +def int_mips_fcne_d : ClangBuiltin<"__builtin_msa_fcne_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcor_w : GCCBuiltin<"__builtin_msa_fcor_w">, +def int_mips_fcor_w : ClangBuiltin<"__builtin_msa_fcor_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcor_d : GCCBuiltin<"__builtin_msa_fcor_d">, +def int_mips_fcor_d : ClangBuiltin<"__builtin_msa_fcor_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcueq_w : GCCBuiltin<"__builtin_msa_fcueq_w">, +def int_mips_fcueq_w : ClangBuiltin<"__builtin_msa_fcueq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcueq_d : GCCBuiltin<"__builtin_msa_fcueq_d">, +def int_mips_fcueq_d : ClangBuiltin<"__builtin_msa_fcueq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcule_w : GCCBuiltin<"__builtin_msa_fcule_w">, +def int_mips_fcule_w : ClangBuiltin<"__builtin_msa_fcule_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcule_d : GCCBuiltin<"__builtin_msa_fcule_d">, +def int_mips_fcule_d : ClangBuiltin<"__builtin_msa_fcule_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcult_w : GCCBuiltin<"__builtin_msa_fcult_w">, +def int_mips_fcult_w : ClangBuiltin<"__builtin_msa_fcult_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcult_d : GCCBuiltin<"__builtin_msa_fcult_d">, +def int_mips_fcult_d : ClangBuiltin<"__builtin_msa_fcult_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcun_w : GCCBuiltin<"__builtin_msa_fcun_w">, +def int_mips_fcun_w : ClangBuiltin<"__builtin_msa_fcun_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcun_d : GCCBuiltin<"__builtin_msa_fcun_d">, +def int_mips_fcun_d : ClangBuiltin<"__builtin_msa_fcun_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fcune_w : GCCBuiltin<"__builtin_msa_fcune_w">, +def int_mips_fcune_w : ClangBuiltin<"__builtin_msa_fcune_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fcune_d : GCCBuiltin<"__builtin_msa_fcune_d">, +def int_mips_fcune_d : ClangBuiltin<"__builtin_msa_fcune_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fdiv_w : GCCBuiltin<"__builtin_msa_fdiv_w">, +def int_mips_fdiv_w : ClangBuiltin<"__builtin_msa_fdiv_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fdiv_d : GCCBuiltin<"__builtin_msa_fdiv_d">, +def int_mips_fdiv_d : ClangBuiltin<"__builtin_msa_fdiv_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fexdo_h : GCCBuiltin<"__builtin_msa_fexdo_h">, +def int_mips_fexdo_h : ClangBuiltin<"__builtin_msa_fexdo_h">, Intrinsic<[llvm_v8f16_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fexdo_w : GCCBuiltin<"__builtin_msa_fexdo_w">, +def int_mips_fexdo_w : ClangBuiltin<"__builtin_msa_fexdo_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fexp2_w : GCCBuiltin<"__builtin_msa_fexp2_w">, +def int_mips_fexp2_w : ClangBuiltin<"__builtin_msa_fexp2_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_fexp2_d : GCCBuiltin<"__builtin_msa_fexp2_d">, +def int_mips_fexp2_d : ClangBuiltin<"__builtin_msa_fexp2_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_fexupl_w : GCCBuiltin<"__builtin_msa_fexupl_w">, +def int_mips_fexupl_w : ClangBuiltin<"__builtin_msa_fexupl_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v8f16_ty], [IntrNoMem]>; -def int_mips_fexupl_d : GCCBuiltin<"__builtin_msa_fexupl_d">, +def int_mips_fexupl_d : ClangBuiltin<"__builtin_msa_fexupl_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fexupr_w : GCCBuiltin<"__builtin_msa_fexupr_w">, +def int_mips_fexupr_w : ClangBuiltin<"__builtin_msa_fexupr_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v8f16_ty], [IntrNoMem]>; -def int_mips_fexupr_d : GCCBuiltin<"__builtin_msa_fexupr_d">, +def int_mips_fexupr_d : ClangBuiltin<"__builtin_msa_fexupr_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ffint_s_w : GCCBuiltin<"__builtin_msa_ffint_s_w">, +def int_mips_ffint_s_w : ClangBuiltin<"__builtin_msa_ffint_s_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ffint_s_d : GCCBuiltin<"__builtin_msa_ffint_s_d">, +def int_mips_ffint_s_d : ClangBuiltin<"__builtin_msa_ffint_s_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ffint_u_w : GCCBuiltin<"__builtin_msa_ffint_u_w">, +def int_mips_ffint_u_w : ClangBuiltin<"__builtin_msa_ffint_u_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ffint_u_d : GCCBuiltin<"__builtin_msa_ffint_u_d">, +def int_mips_ffint_u_d : ClangBuiltin<"__builtin_msa_ffint_u_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ffql_w : GCCBuiltin<"__builtin_msa_ffql_w">, +def int_mips_ffql_w : ClangBuiltin<"__builtin_msa_ffql_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ffql_d : GCCBuiltin<"__builtin_msa_ffql_d">, +def int_mips_ffql_d : ClangBuiltin<"__builtin_msa_ffql_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ffqr_w : GCCBuiltin<"__builtin_msa_ffqr_w">, +def int_mips_ffqr_w : ClangBuiltin<"__builtin_msa_ffqr_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ffqr_d : GCCBuiltin<"__builtin_msa_ffqr_d">, +def int_mips_ffqr_d : ClangBuiltin<"__builtin_msa_ffqr_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_fill_b : GCCBuiltin<"__builtin_msa_fill_b">, +def int_mips_fill_b : ClangBuiltin<"__builtin_msa_fill_b">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_fill_h : GCCBuiltin<"__builtin_msa_fill_h">, +def int_mips_fill_h : ClangBuiltin<"__builtin_msa_fill_h">, Intrinsic<[llvm_v8i16_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_fill_w : GCCBuiltin<"__builtin_msa_fill_w">, +def int_mips_fill_w : ClangBuiltin<"__builtin_msa_fill_w">, Intrinsic<[llvm_v4i32_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_mips_fill_d : GCCBuiltin<"__builtin_msa_fill_d">, +def int_mips_fill_d : ClangBuiltin<"__builtin_msa_fill_d">, Intrinsic<[llvm_v2i64_ty], [llvm_i64_ty], [IntrNoMem]>; -def int_mips_flog2_w : GCCBuiltin<"__builtin_msa_flog2_w">, +def int_mips_flog2_w : ClangBuiltin<"__builtin_msa_flog2_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_flog2_d : GCCBuiltin<"__builtin_msa_flog2_d">, +def int_mips_flog2_d : ClangBuiltin<"__builtin_msa_flog2_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmadd_w : GCCBuiltin<"__builtin_msa_fmadd_w">, +def int_mips_fmadd_w : ClangBuiltin<"__builtin_msa_fmadd_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmadd_d : GCCBuiltin<"__builtin_msa_fmadd_d">, +def int_mips_fmadd_d : ClangBuiltin<"__builtin_msa_fmadd_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmax_w : GCCBuiltin<"__builtin_msa_fmax_w">, +def int_mips_fmax_w : ClangBuiltin<"__builtin_msa_fmax_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmax_d : GCCBuiltin<"__builtin_msa_fmax_d">, +def int_mips_fmax_d : ClangBuiltin<"__builtin_msa_fmax_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmax_a_w : GCCBuiltin<"__builtin_msa_fmax_a_w">, +def int_mips_fmax_a_w : ClangBuiltin<"__builtin_msa_fmax_a_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmax_a_d : GCCBuiltin<"__builtin_msa_fmax_a_d">, +def int_mips_fmax_a_d : ClangBuiltin<"__builtin_msa_fmax_a_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmin_w : GCCBuiltin<"__builtin_msa_fmin_w">, +def int_mips_fmin_w : ClangBuiltin<"__builtin_msa_fmin_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmin_d : GCCBuiltin<"__builtin_msa_fmin_d">, +def int_mips_fmin_d : ClangBuiltin<"__builtin_msa_fmin_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmin_a_w : GCCBuiltin<"__builtin_msa_fmin_a_w">, +def int_mips_fmin_a_w : ClangBuiltin<"__builtin_msa_fmin_a_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmin_a_d : GCCBuiltin<"__builtin_msa_fmin_a_d">, +def int_mips_fmin_a_d : ClangBuiltin<"__builtin_msa_fmin_a_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmsub_w : GCCBuiltin<"__builtin_msa_fmsub_w">, +def int_mips_fmsub_w : ClangBuiltin<"__builtin_msa_fmsub_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmsub_d : GCCBuiltin<"__builtin_msa_fmsub_d">, +def int_mips_fmsub_d : ClangBuiltin<"__builtin_msa_fmsub_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fmul_w : GCCBuiltin<"__builtin_msa_fmul_w">, +def int_mips_fmul_w : ClangBuiltin<"__builtin_msa_fmul_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fmul_d : GCCBuiltin<"__builtin_msa_fmul_d">, +def int_mips_fmul_d : ClangBuiltin<"__builtin_msa_fmul_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_frint_w : GCCBuiltin<"__builtin_msa_frint_w">, +def int_mips_frint_w : ClangBuiltin<"__builtin_msa_frint_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_frint_d : GCCBuiltin<"__builtin_msa_frint_d">, +def int_mips_frint_d : ClangBuiltin<"__builtin_msa_frint_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_frcp_w : GCCBuiltin<"__builtin_msa_frcp_w">, +def int_mips_frcp_w : ClangBuiltin<"__builtin_msa_frcp_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_frcp_d : GCCBuiltin<"__builtin_msa_frcp_d">, +def int_mips_frcp_d : ClangBuiltin<"__builtin_msa_frcp_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_frsqrt_w : GCCBuiltin<"__builtin_msa_frsqrt_w">, +def int_mips_frsqrt_w : ClangBuiltin<"__builtin_msa_frsqrt_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_frsqrt_d : GCCBuiltin<"__builtin_msa_frsqrt_d">, +def int_mips_frsqrt_d : ClangBuiltin<"__builtin_msa_frsqrt_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsaf_w : GCCBuiltin<"__builtin_msa_fsaf_w">, +def int_mips_fsaf_w : ClangBuiltin<"__builtin_msa_fsaf_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsaf_d : GCCBuiltin<"__builtin_msa_fsaf_d">, +def int_mips_fsaf_d : ClangBuiltin<"__builtin_msa_fsaf_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fseq_w : GCCBuiltin<"__builtin_msa_fseq_w">, +def int_mips_fseq_w : ClangBuiltin<"__builtin_msa_fseq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fseq_d : GCCBuiltin<"__builtin_msa_fseq_d">, +def int_mips_fseq_d : ClangBuiltin<"__builtin_msa_fseq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsle_w : GCCBuiltin<"__builtin_msa_fsle_w">, +def int_mips_fsle_w : ClangBuiltin<"__builtin_msa_fsle_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsle_d : GCCBuiltin<"__builtin_msa_fsle_d">, +def int_mips_fsle_d : ClangBuiltin<"__builtin_msa_fsle_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fslt_w : GCCBuiltin<"__builtin_msa_fslt_w">, +def int_mips_fslt_w : ClangBuiltin<"__builtin_msa_fslt_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fslt_d : GCCBuiltin<"__builtin_msa_fslt_d">, +def int_mips_fslt_d : ClangBuiltin<"__builtin_msa_fslt_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsne_w : GCCBuiltin<"__builtin_msa_fsne_w">, +def int_mips_fsne_w : ClangBuiltin<"__builtin_msa_fsne_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsne_d : GCCBuiltin<"__builtin_msa_fsne_d">, +def int_mips_fsne_d : ClangBuiltin<"__builtin_msa_fsne_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsor_w : GCCBuiltin<"__builtin_msa_fsor_w">, +def int_mips_fsor_w : ClangBuiltin<"__builtin_msa_fsor_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsor_d : GCCBuiltin<"__builtin_msa_fsor_d">, +def int_mips_fsor_d : ClangBuiltin<"__builtin_msa_fsor_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsqrt_w : GCCBuiltin<"__builtin_msa_fsqrt_w">, +def int_mips_fsqrt_w : ClangBuiltin<"__builtin_msa_fsqrt_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsqrt_d : GCCBuiltin<"__builtin_msa_fsqrt_d">, +def int_mips_fsqrt_d : ClangBuiltin<"__builtin_msa_fsqrt_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsub_w : GCCBuiltin<"__builtin_msa_fsub_w">, +def int_mips_fsub_w : ClangBuiltin<"__builtin_msa_fsub_w">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsub_d : GCCBuiltin<"__builtin_msa_fsub_d">, +def int_mips_fsub_d : ClangBuiltin<"__builtin_msa_fsub_d">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsueq_w : GCCBuiltin<"__builtin_msa_fsueq_w">, +def int_mips_fsueq_w : ClangBuiltin<"__builtin_msa_fsueq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsueq_d : GCCBuiltin<"__builtin_msa_fsueq_d">, +def int_mips_fsueq_d : ClangBuiltin<"__builtin_msa_fsueq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsule_w : GCCBuiltin<"__builtin_msa_fsule_w">, +def int_mips_fsule_w : ClangBuiltin<"__builtin_msa_fsule_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsule_d : GCCBuiltin<"__builtin_msa_fsule_d">, +def int_mips_fsule_d : ClangBuiltin<"__builtin_msa_fsule_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsult_w : GCCBuiltin<"__builtin_msa_fsult_w">, +def int_mips_fsult_w : ClangBuiltin<"__builtin_msa_fsult_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsult_d : GCCBuiltin<"__builtin_msa_fsult_d">, +def int_mips_fsult_d : ClangBuiltin<"__builtin_msa_fsult_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsun_w : GCCBuiltin<"__builtin_msa_fsun_w">, +def int_mips_fsun_w : ClangBuiltin<"__builtin_msa_fsun_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsun_d : GCCBuiltin<"__builtin_msa_fsun_d">, +def int_mips_fsun_d : ClangBuiltin<"__builtin_msa_fsun_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_fsune_w : GCCBuiltin<"__builtin_msa_fsune_w">, +def int_mips_fsune_w : ClangBuiltin<"__builtin_msa_fsune_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_fsune_d : GCCBuiltin<"__builtin_msa_fsune_d">, +def int_mips_fsune_d : ClangBuiltin<"__builtin_msa_fsune_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_ftint_s_w : GCCBuiltin<"__builtin_msa_ftint_s_w">, +def int_mips_ftint_s_w : ClangBuiltin<"__builtin_msa_ftint_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ftint_s_d : GCCBuiltin<"__builtin_msa_ftint_s_d">, +def int_mips_ftint_s_d : ClangBuiltin<"__builtin_msa_ftint_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_ftint_u_w : GCCBuiltin<"__builtin_msa_ftint_u_w">, +def int_mips_ftint_u_w : ClangBuiltin<"__builtin_msa_ftint_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ftint_u_d : GCCBuiltin<"__builtin_msa_ftint_u_d">, +def int_mips_ftint_u_d : ClangBuiltin<"__builtin_msa_ftint_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_ftq_h : GCCBuiltin<"__builtin_msa_ftq_h">, +def int_mips_ftq_h : ClangBuiltin<"__builtin_msa_ftq_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ftq_w : GCCBuiltin<"__builtin_msa_ftq_w">, +def int_mips_ftq_w : ClangBuiltin<"__builtin_msa_ftq_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_ftrunc_s_w : GCCBuiltin<"__builtin_msa_ftrunc_s_w">, +def int_mips_ftrunc_s_w : ClangBuiltin<"__builtin_msa_ftrunc_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ftrunc_s_d : GCCBuiltin<"__builtin_msa_ftrunc_s_d">, +def int_mips_ftrunc_s_d : ClangBuiltin<"__builtin_msa_ftrunc_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_ftrunc_u_w : GCCBuiltin<"__builtin_msa_ftrunc_u_w">, +def int_mips_ftrunc_u_w : ClangBuiltin<"__builtin_msa_ftrunc_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_mips_ftrunc_u_d : GCCBuiltin<"__builtin_msa_ftrunc_u_d">, +def int_mips_ftrunc_u_d : ClangBuiltin<"__builtin_msa_ftrunc_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; -def int_mips_hadd_s_h : GCCBuiltin<"__builtin_msa_hadd_s_h">, +def int_mips_hadd_s_h : ClangBuiltin<"__builtin_msa_hadd_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_hadd_s_w : GCCBuiltin<"__builtin_msa_hadd_s_w">, +def int_mips_hadd_s_w : ClangBuiltin<"__builtin_msa_hadd_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_hadd_s_d : GCCBuiltin<"__builtin_msa_hadd_s_d">, +def int_mips_hadd_s_d : ClangBuiltin<"__builtin_msa_hadd_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_hadd_u_h : GCCBuiltin<"__builtin_msa_hadd_u_h">, +def int_mips_hadd_u_h : ClangBuiltin<"__builtin_msa_hadd_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_hadd_u_w : GCCBuiltin<"__builtin_msa_hadd_u_w">, +def int_mips_hadd_u_w : ClangBuiltin<"__builtin_msa_hadd_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_hadd_u_d : GCCBuiltin<"__builtin_msa_hadd_u_d">, +def int_mips_hadd_u_d : ClangBuiltin<"__builtin_msa_hadd_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_hsub_s_h : GCCBuiltin<"__builtin_msa_hsub_s_h">, +def int_mips_hsub_s_h : ClangBuiltin<"__builtin_msa_hsub_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_hsub_s_w : GCCBuiltin<"__builtin_msa_hsub_s_w">, +def int_mips_hsub_s_w : ClangBuiltin<"__builtin_msa_hsub_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_hsub_s_d : GCCBuiltin<"__builtin_msa_hsub_s_d">, +def int_mips_hsub_s_d : ClangBuiltin<"__builtin_msa_hsub_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_hsub_u_h : GCCBuiltin<"__builtin_msa_hsub_u_h">, +def int_mips_hsub_u_h : ClangBuiltin<"__builtin_msa_hsub_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_hsub_u_w : GCCBuiltin<"__builtin_msa_hsub_u_w">, +def int_mips_hsub_u_w : ClangBuiltin<"__builtin_msa_hsub_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_hsub_u_d : GCCBuiltin<"__builtin_msa_hsub_u_d">, +def int_mips_hsub_u_d : ClangBuiltin<"__builtin_msa_hsub_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ilvev_b : GCCBuiltin<"__builtin_msa_ilvev_b">, +def int_mips_ilvev_b : ClangBuiltin<"__builtin_msa_ilvev_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ilvev_h : GCCBuiltin<"__builtin_msa_ilvev_h">, +def int_mips_ilvev_h : ClangBuiltin<"__builtin_msa_ilvev_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ilvev_w : GCCBuiltin<"__builtin_msa_ilvev_w">, +def int_mips_ilvev_w : ClangBuiltin<"__builtin_msa_ilvev_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ilvev_d : GCCBuiltin<"__builtin_msa_ilvev_d">, +def int_mips_ilvev_d : ClangBuiltin<"__builtin_msa_ilvev_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ilvl_b : GCCBuiltin<"__builtin_msa_ilvl_b">, +def int_mips_ilvl_b : ClangBuiltin<"__builtin_msa_ilvl_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ilvl_h : GCCBuiltin<"__builtin_msa_ilvl_h">, +def int_mips_ilvl_h : ClangBuiltin<"__builtin_msa_ilvl_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ilvl_w : GCCBuiltin<"__builtin_msa_ilvl_w">, +def int_mips_ilvl_w : ClangBuiltin<"__builtin_msa_ilvl_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ilvl_d : GCCBuiltin<"__builtin_msa_ilvl_d">, +def int_mips_ilvl_d : ClangBuiltin<"__builtin_msa_ilvl_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ilvod_b : GCCBuiltin<"__builtin_msa_ilvod_b">, +def int_mips_ilvod_b : ClangBuiltin<"__builtin_msa_ilvod_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ilvod_h : GCCBuiltin<"__builtin_msa_ilvod_h">, +def int_mips_ilvod_h : ClangBuiltin<"__builtin_msa_ilvod_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ilvod_w : GCCBuiltin<"__builtin_msa_ilvod_w">, +def int_mips_ilvod_w : ClangBuiltin<"__builtin_msa_ilvod_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ilvod_d : GCCBuiltin<"__builtin_msa_ilvod_d">, +def int_mips_ilvod_d : ClangBuiltin<"__builtin_msa_ilvod_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_ilvr_b : GCCBuiltin<"__builtin_msa_ilvr_b">, +def int_mips_ilvr_b : ClangBuiltin<"__builtin_msa_ilvr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ilvr_h : GCCBuiltin<"__builtin_msa_ilvr_h">, +def int_mips_ilvr_h : ClangBuiltin<"__builtin_msa_ilvr_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_ilvr_w : GCCBuiltin<"__builtin_msa_ilvr_w">, +def int_mips_ilvr_w : ClangBuiltin<"__builtin_msa_ilvr_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_ilvr_d : GCCBuiltin<"__builtin_msa_ilvr_d">, +def int_mips_ilvr_d : ClangBuiltin<"__builtin_msa_ilvr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_insert_b : GCCBuiltin<"__builtin_msa_insert_b">, +def int_mips_insert_b : ClangBuiltin<"__builtin_msa_insert_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_insert_h : GCCBuiltin<"__builtin_msa_insert_h">, +def int_mips_insert_h : ClangBuiltin<"__builtin_msa_insert_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_insert_w : GCCBuiltin<"__builtin_msa_insert_w">, +def int_mips_insert_w : ClangBuiltin<"__builtin_msa_insert_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_insert_d : GCCBuiltin<"__builtin_msa_insert_d">, +def int_mips_insert_d : ClangBuiltin<"__builtin_msa_insert_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; -def int_mips_insve_b : GCCBuiltin<"__builtin_msa_insve_b">, +def int_mips_insve_b : ClangBuiltin<"__builtin_msa_insve_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_insve_h : GCCBuiltin<"__builtin_msa_insve_h">, +def int_mips_insve_h : ClangBuiltin<"__builtin_msa_insve_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_insve_w : GCCBuiltin<"__builtin_msa_insve_w">, +def int_mips_insve_w : ClangBuiltin<"__builtin_msa_insve_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_insve_d : GCCBuiltin<"__builtin_msa_insve_d">, +def int_mips_insve_d : ClangBuiltin<"__builtin_msa_insve_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_ld_b : GCCBuiltin<"__builtin_msa_ld_b">, +def int_mips_ld_b : ClangBuiltin<"__builtin_msa_ld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ld_h : GCCBuiltin<"__builtin_msa_ld_h">, +def int_mips_ld_h : ClangBuiltin<"__builtin_msa_ld_h">, Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ld_w : GCCBuiltin<"__builtin_msa_ld_w">, +def int_mips_ld_w : ClangBuiltin<"__builtin_msa_ld_w">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ld_d : GCCBuiltin<"__builtin_msa_ld_d">, +def int_mips_ld_d : ClangBuiltin<"__builtin_msa_ld_d">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ldr_d : GCCBuiltin<"__builtin_msa_ldr_d">, +def int_mips_ldr_d : ClangBuiltin<"__builtin_msa_ldr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ldr_w : GCCBuiltin<"__builtin_msa_ldr_w">, +def int_mips_ldr_w : ClangBuiltin<"__builtin_msa_ldr_w">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_mips_ldi_b : GCCBuiltin<"__builtin_msa_ldi_b">, +def int_mips_ldi_b : ClangBuiltin<"__builtin_msa_ldi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; -def int_mips_ldi_h : GCCBuiltin<"__builtin_msa_ldi_h">, +def int_mips_ldi_h : ClangBuiltin<"__builtin_msa_ldi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; -def int_mips_ldi_w : GCCBuiltin<"__builtin_msa_ldi_w">, +def int_mips_ldi_w : ClangBuiltin<"__builtin_msa_ldi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; -def int_mips_ldi_d : GCCBuiltin<"__builtin_msa_ldi_d">, +def int_mips_ldi_d : ClangBuiltin<"__builtin_msa_ldi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; // This instruction is part of the MSA spec but it does not share the // __builtin_msa prefix because it operates on the GPR registers. -def int_mips_lsa : GCCBuiltin<"__builtin_mips_lsa">, +def int_mips_lsa : ClangBuiltin<"__builtin_mips_lsa">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_madd_q_h : GCCBuiltin<"__builtin_msa_madd_q_h">, +def int_mips_madd_q_h : ClangBuiltin<"__builtin_msa_madd_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_madd_q_w : GCCBuiltin<"__builtin_msa_madd_q_w">, +def int_mips_madd_q_w : ClangBuiltin<"__builtin_msa_madd_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_maddr_q_h : GCCBuiltin<"__builtin_msa_maddr_q_h">, +def int_mips_maddr_q_h : ClangBuiltin<"__builtin_msa_maddr_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_maddr_q_w : GCCBuiltin<"__builtin_msa_maddr_q_w">, +def int_mips_maddr_q_w : ClangBuiltin<"__builtin_msa_maddr_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_maddv_b : GCCBuiltin<"__builtin_msa_maddv_b">, +def int_mips_maddv_b : ClangBuiltin<"__builtin_msa_maddv_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_maddv_h : GCCBuiltin<"__builtin_msa_maddv_h">, +def int_mips_maddv_h : ClangBuiltin<"__builtin_msa_maddv_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_maddv_w : GCCBuiltin<"__builtin_msa_maddv_w">, +def int_mips_maddv_w : ClangBuiltin<"__builtin_msa_maddv_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_maddv_d : GCCBuiltin<"__builtin_msa_maddv_d">, +def int_mips_maddv_d : ClangBuiltin<"__builtin_msa_maddv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_max_a_b : GCCBuiltin<"__builtin_msa_max_a_b">, +def int_mips_max_a_b : ClangBuiltin<"__builtin_msa_max_a_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_max_a_h : GCCBuiltin<"__builtin_msa_max_a_h">, +def int_mips_max_a_h : ClangBuiltin<"__builtin_msa_max_a_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_max_a_w : GCCBuiltin<"__builtin_msa_max_a_w">, +def int_mips_max_a_w : ClangBuiltin<"__builtin_msa_max_a_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_max_a_d : GCCBuiltin<"__builtin_msa_max_a_d">, +def int_mips_max_a_d : ClangBuiltin<"__builtin_msa_max_a_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_max_s_b : GCCBuiltin<"__builtin_msa_max_s_b">, +def int_mips_max_s_b : ClangBuiltin<"__builtin_msa_max_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_max_s_h : GCCBuiltin<"__builtin_msa_max_s_h">, +def int_mips_max_s_h : ClangBuiltin<"__builtin_msa_max_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_max_s_w : GCCBuiltin<"__builtin_msa_max_s_w">, +def int_mips_max_s_w : ClangBuiltin<"__builtin_msa_max_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_max_s_d : GCCBuiltin<"__builtin_msa_max_s_d">, +def int_mips_max_s_d : ClangBuiltin<"__builtin_msa_max_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_max_u_b : GCCBuiltin<"__builtin_msa_max_u_b">, +def int_mips_max_u_b : ClangBuiltin<"__builtin_msa_max_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_max_u_h : GCCBuiltin<"__builtin_msa_max_u_h">, +def int_mips_max_u_h : ClangBuiltin<"__builtin_msa_max_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_max_u_w : GCCBuiltin<"__builtin_msa_max_u_w">, +def int_mips_max_u_w : ClangBuiltin<"__builtin_msa_max_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_max_u_d : GCCBuiltin<"__builtin_msa_max_u_d">, +def int_mips_max_u_d : ClangBuiltin<"__builtin_msa_max_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_maxi_s_b : GCCBuiltin<"__builtin_msa_maxi_s_b">, +def int_mips_maxi_s_b : ClangBuiltin<"__builtin_msa_maxi_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_s_h : GCCBuiltin<"__builtin_msa_maxi_s_h">, +def int_mips_maxi_s_h : ClangBuiltin<"__builtin_msa_maxi_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_s_w : GCCBuiltin<"__builtin_msa_maxi_s_w">, +def int_mips_maxi_s_w : ClangBuiltin<"__builtin_msa_maxi_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_s_d : GCCBuiltin<"__builtin_msa_maxi_s_d">, +def int_mips_maxi_s_d : ClangBuiltin<"__builtin_msa_maxi_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_u_b : GCCBuiltin<"__builtin_msa_maxi_u_b">, +def int_mips_maxi_u_b : ClangBuiltin<"__builtin_msa_maxi_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_u_h : GCCBuiltin<"__builtin_msa_maxi_u_h">, +def int_mips_maxi_u_h : ClangBuiltin<"__builtin_msa_maxi_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_u_w : GCCBuiltin<"__builtin_msa_maxi_u_w">, +def int_mips_maxi_u_w : ClangBuiltin<"__builtin_msa_maxi_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_maxi_u_d : GCCBuiltin<"__builtin_msa_maxi_u_d">, +def int_mips_maxi_u_d : ClangBuiltin<"__builtin_msa_maxi_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_min_a_b : GCCBuiltin<"__builtin_msa_min_a_b">, +def int_mips_min_a_b : ClangBuiltin<"__builtin_msa_min_a_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_min_a_h : GCCBuiltin<"__builtin_msa_min_a_h">, +def int_mips_min_a_h : ClangBuiltin<"__builtin_msa_min_a_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_min_a_w : GCCBuiltin<"__builtin_msa_min_a_w">, +def int_mips_min_a_w : ClangBuiltin<"__builtin_msa_min_a_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_min_a_d : GCCBuiltin<"__builtin_msa_min_a_d">, +def int_mips_min_a_d : ClangBuiltin<"__builtin_msa_min_a_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_min_s_b : GCCBuiltin<"__builtin_msa_min_s_b">, +def int_mips_min_s_b : ClangBuiltin<"__builtin_msa_min_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_min_s_h : GCCBuiltin<"__builtin_msa_min_s_h">, +def int_mips_min_s_h : ClangBuiltin<"__builtin_msa_min_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_min_s_w : GCCBuiltin<"__builtin_msa_min_s_w">, +def int_mips_min_s_w : ClangBuiltin<"__builtin_msa_min_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_min_s_d : GCCBuiltin<"__builtin_msa_min_s_d">, +def int_mips_min_s_d : ClangBuiltin<"__builtin_msa_min_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_min_u_b : GCCBuiltin<"__builtin_msa_min_u_b">, +def int_mips_min_u_b : ClangBuiltin<"__builtin_msa_min_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_min_u_h : GCCBuiltin<"__builtin_msa_min_u_h">, +def int_mips_min_u_h : ClangBuiltin<"__builtin_msa_min_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_min_u_w : GCCBuiltin<"__builtin_msa_min_u_w">, +def int_mips_min_u_w : ClangBuiltin<"__builtin_msa_min_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_min_u_d : GCCBuiltin<"__builtin_msa_min_u_d">, +def int_mips_min_u_d : ClangBuiltin<"__builtin_msa_min_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_mini_s_b : GCCBuiltin<"__builtin_msa_mini_s_b">, +def int_mips_mini_s_b : ClangBuiltin<"__builtin_msa_mini_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_s_h : GCCBuiltin<"__builtin_msa_mini_s_h">, +def int_mips_mini_s_h : ClangBuiltin<"__builtin_msa_mini_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_s_w : GCCBuiltin<"__builtin_msa_mini_s_w">, +def int_mips_mini_s_w : ClangBuiltin<"__builtin_msa_mini_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_s_d : GCCBuiltin<"__builtin_msa_mini_s_d">, +def int_mips_mini_s_d : ClangBuiltin<"__builtin_msa_mini_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_u_b : GCCBuiltin<"__builtin_msa_mini_u_b">, +def int_mips_mini_u_b : ClangBuiltin<"__builtin_msa_mini_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_u_h : GCCBuiltin<"__builtin_msa_mini_u_h">, +def int_mips_mini_u_h : ClangBuiltin<"__builtin_msa_mini_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_u_w : GCCBuiltin<"__builtin_msa_mini_u_w">, +def int_mips_mini_u_w : ClangBuiltin<"__builtin_msa_mini_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mini_u_d : GCCBuiltin<"__builtin_msa_mini_u_d">, +def int_mips_mini_u_d : ClangBuiltin<"__builtin_msa_mini_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_mod_s_b : GCCBuiltin<"__builtin_msa_mod_s_b">, +def int_mips_mod_s_b : ClangBuiltin<"__builtin_msa_mod_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_mod_s_h : GCCBuiltin<"__builtin_msa_mod_s_h">, +def int_mips_mod_s_h : ClangBuiltin<"__builtin_msa_mod_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_mod_s_w : GCCBuiltin<"__builtin_msa_mod_s_w">, +def int_mips_mod_s_w : ClangBuiltin<"__builtin_msa_mod_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_mod_s_d : GCCBuiltin<"__builtin_msa_mod_s_d">, +def int_mips_mod_s_d : ClangBuiltin<"__builtin_msa_mod_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_mod_u_b : GCCBuiltin<"__builtin_msa_mod_u_b">, +def int_mips_mod_u_b : ClangBuiltin<"__builtin_msa_mod_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_mod_u_h : GCCBuiltin<"__builtin_msa_mod_u_h">, +def int_mips_mod_u_h : ClangBuiltin<"__builtin_msa_mod_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_mod_u_w : GCCBuiltin<"__builtin_msa_mod_u_w">, +def int_mips_mod_u_w : ClangBuiltin<"__builtin_msa_mod_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_mod_u_d : GCCBuiltin<"__builtin_msa_mod_u_d">, +def int_mips_mod_u_d : ClangBuiltin<"__builtin_msa_mod_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_move_v : GCCBuiltin<"__builtin_msa_move_v">, +def int_mips_move_v : ClangBuiltin<"__builtin_msa_move_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_msub_q_h : GCCBuiltin<"__builtin_msa_msub_q_h">, +def int_mips_msub_q_h : ClangBuiltin<"__builtin_msa_msub_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_msub_q_w : GCCBuiltin<"__builtin_msa_msub_q_w">, +def int_mips_msub_q_w : ClangBuiltin<"__builtin_msa_msub_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_msubr_q_h : GCCBuiltin<"__builtin_msa_msubr_q_h">, +def int_mips_msubr_q_h : ClangBuiltin<"__builtin_msa_msubr_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_msubr_q_w : GCCBuiltin<"__builtin_msa_msubr_q_w">, +def int_mips_msubr_q_w : ClangBuiltin<"__builtin_msa_msubr_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_msubv_b : GCCBuiltin<"__builtin_msa_msubv_b">, +def int_mips_msubv_b : ClangBuiltin<"__builtin_msa_msubv_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_msubv_h : GCCBuiltin<"__builtin_msa_msubv_h">, +def int_mips_msubv_h : ClangBuiltin<"__builtin_msa_msubv_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_msubv_w : GCCBuiltin<"__builtin_msa_msubv_w">, +def int_mips_msubv_w : ClangBuiltin<"__builtin_msa_msubv_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_msubv_d : GCCBuiltin<"__builtin_msa_msubv_d">, +def int_mips_msubv_d : ClangBuiltin<"__builtin_msa_msubv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_mul_q_h : GCCBuiltin<"__builtin_msa_mul_q_h">, +def int_mips_mul_q_h : ClangBuiltin<"__builtin_msa_mul_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_mul_q_w : GCCBuiltin<"__builtin_msa_mul_q_w">, +def int_mips_mul_q_w : ClangBuiltin<"__builtin_msa_mul_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_mulr_q_h : GCCBuiltin<"__builtin_msa_mulr_q_h">, +def int_mips_mulr_q_h : ClangBuiltin<"__builtin_msa_mulr_q_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_mulr_q_w : GCCBuiltin<"__builtin_msa_mulr_q_w">, +def int_mips_mulr_q_w : ClangBuiltin<"__builtin_msa_mulr_q_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_mulv_b : GCCBuiltin<"__builtin_msa_mulv_b">, +def int_mips_mulv_b : ClangBuiltin<"__builtin_msa_mulv_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_mulv_h : GCCBuiltin<"__builtin_msa_mulv_h">, +def int_mips_mulv_h : ClangBuiltin<"__builtin_msa_mulv_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_mulv_w : GCCBuiltin<"__builtin_msa_mulv_w">, +def int_mips_mulv_w : ClangBuiltin<"__builtin_msa_mulv_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_mulv_d : GCCBuiltin<"__builtin_msa_mulv_d">, +def int_mips_mulv_d : ClangBuiltin<"__builtin_msa_mulv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_nloc_b : GCCBuiltin<"__builtin_msa_nloc_b">, +def int_mips_nloc_b : ClangBuiltin<"__builtin_msa_nloc_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_nloc_h : GCCBuiltin<"__builtin_msa_nloc_h">, +def int_mips_nloc_h : ClangBuiltin<"__builtin_msa_nloc_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_nloc_w : GCCBuiltin<"__builtin_msa_nloc_w">, +def int_mips_nloc_w : ClangBuiltin<"__builtin_msa_nloc_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_nloc_d : GCCBuiltin<"__builtin_msa_nloc_d">, +def int_mips_nloc_d : ClangBuiltin<"__builtin_msa_nloc_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_nlzc_b : GCCBuiltin<"__builtin_msa_nlzc_b">, +def int_mips_nlzc_b : ClangBuiltin<"__builtin_msa_nlzc_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_nlzc_h : GCCBuiltin<"__builtin_msa_nlzc_h">, +def int_mips_nlzc_h : ClangBuiltin<"__builtin_msa_nlzc_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_nlzc_w : GCCBuiltin<"__builtin_msa_nlzc_w">, +def int_mips_nlzc_w : ClangBuiltin<"__builtin_msa_nlzc_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_nlzc_d : GCCBuiltin<"__builtin_msa_nlzc_d">, +def int_mips_nlzc_d : ClangBuiltin<"__builtin_msa_nlzc_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_nor_v : GCCBuiltin<"__builtin_msa_nor_v">, +def int_mips_nor_v : ClangBuiltin<"__builtin_msa_nor_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_nori_b : GCCBuiltin<"__builtin_msa_nori_b">, +def int_mips_nori_b : ClangBuiltin<"__builtin_msa_nori_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_or_v : GCCBuiltin<"__builtin_msa_or_v">, +def int_mips_or_v : ClangBuiltin<"__builtin_msa_or_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_ori_b : GCCBuiltin<"__builtin_msa_ori_b">, +def int_mips_ori_b : ClangBuiltin<"__builtin_msa_ori_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_pckev_b : GCCBuiltin<"__builtin_msa_pckev_b">, +def int_mips_pckev_b : ClangBuiltin<"__builtin_msa_pckev_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_pckev_h : GCCBuiltin<"__builtin_msa_pckev_h">, +def int_mips_pckev_h : ClangBuiltin<"__builtin_msa_pckev_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_pckev_w : GCCBuiltin<"__builtin_msa_pckev_w">, +def int_mips_pckev_w : ClangBuiltin<"__builtin_msa_pckev_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_pckev_d : GCCBuiltin<"__builtin_msa_pckev_d">, +def int_mips_pckev_d : ClangBuiltin<"__builtin_msa_pckev_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_pckod_b : GCCBuiltin<"__builtin_msa_pckod_b">, +def int_mips_pckod_b : ClangBuiltin<"__builtin_msa_pckod_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_pckod_h : GCCBuiltin<"__builtin_msa_pckod_h">, +def int_mips_pckod_h : ClangBuiltin<"__builtin_msa_pckod_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_pckod_w : GCCBuiltin<"__builtin_msa_pckod_w">, +def int_mips_pckod_w : ClangBuiltin<"__builtin_msa_pckod_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_pckod_d : GCCBuiltin<"__builtin_msa_pckod_d">, +def int_mips_pckod_d : ClangBuiltin<"__builtin_msa_pckod_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_pcnt_b : GCCBuiltin<"__builtin_msa_pcnt_b">, +def int_mips_pcnt_b : ClangBuiltin<"__builtin_msa_pcnt_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_pcnt_h : GCCBuiltin<"__builtin_msa_pcnt_h">, +def int_mips_pcnt_h : ClangBuiltin<"__builtin_msa_pcnt_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_pcnt_w : GCCBuiltin<"__builtin_msa_pcnt_w">, +def int_mips_pcnt_w : ClangBuiltin<"__builtin_msa_pcnt_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_pcnt_d : GCCBuiltin<"__builtin_msa_pcnt_d">, +def int_mips_pcnt_d : ClangBuiltin<"__builtin_msa_pcnt_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_sat_s_b : GCCBuiltin<"__builtin_msa_sat_s_b">, +def int_mips_sat_s_b : ClangBuiltin<"__builtin_msa_sat_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_s_h : GCCBuiltin<"__builtin_msa_sat_s_h">, +def int_mips_sat_s_h : ClangBuiltin<"__builtin_msa_sat_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_s_w : GCCBuiltin<"__builtin_msa_sat_s_w">, +def int_mips_sat_s_w : ClangBuiltin<"__builtin_msa_sat_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_s_d : GCCBuiltin<"__builtin_msa_sat_s_d">, +def int_mips_sat_s_d : ClangBuiltin<"__builtin_msa_sat_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_u_b : GCCBuiltin<"__builtin_msa_sat_u_b">, +def int_mips_sat_u_b : ClangBuiltin<"__builtin_msa_sat_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_u_h : GCCBuiltin<"__builtin_msa_sat_u_h">, +def int_mips_sat_u_h : ClangBuiltin<"__builtin_msa_sat_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_u_w : GCCBuiltin<"__builtin_msa_sat_u_w">, +def int_mips_sat_u_w : ClangBuiltin<"__builtin_msa_sat_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sat_u_d : GCCBuiltin<"__builtin_msa_sat_u_d">, +def int_mips_sat_u_d : ClangBuiltin<"__builtin_msa_sat_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_shf_b : GCCBuiltin<"__builtin_msa_shf_b">, +def int_mips_shf_b : ClangBuiltin<"__builtin_msa_shf_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_shf_h : GCCBuiltin<"__builtin_msa_shf_h">, +def int_mips_shf_h : ClangBuiltin<"__builtin_msa_shf_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_shf_w : GCCBuiltin<"__builtin_msa_shf_w">, +def int_mips_shf_w : ClangBuiltin<"__builtin_msa_shf_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sld_b : GCCBuiltin<"__builtin_msa_sld_b">, +def int_mips_sld_b : ClangBuiltin<"__builtin_msa_sld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_sld_h : GCCBuiltin<"__builtin_msa_sld_h">, +def int_mips_sld_h : ClangBuiltin<"__builtin_msa_sld_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_sld_w : GCCBuiltin<"__builtin_msa_sld_w">, +def int_mips_sld_w : ClangBuiltin<"__builtin_msa_sld_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_sld_d : GCCBuiltin<"__builtin_msa_sld_d">, +def int_mips_sld_d : ClangBuiltin<"__builtin_msa_sld_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_sldi_b : GCCBuiltin<"__builtin_msa_sldi_b">, +def int_mips_sldi_b : ClangBuiltin<"__builtin_msa_sldi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_sldi_h : GCCBuiltin<"__builtin_msa_sldi_h">, +def int_mips_sldi_h : ClangBuiltin<"__builtin_msa_sldi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_sldi_w : GCCBuiltin<"__builtin_msa_sldi_w">, +def int_mips_sldi_w : ClangBuiltin<"__builtin_msa_sldi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_sldi_d : GCCBuiltin<"__builtin_msa_sldi_d">, +def int_mips_sldi_d : ClangBuiltin<"__builtin_msa_sldi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_mips_sll_b : GCCBuiltin<"__builtin_msa_sll_b">, +def int_mips_sll_b : ClangBuiltin<"__builtin_msa_sll_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_sll_h : GCCBuiltin<"__builtin_msa_sll_h">, +def int_mips_sll_h : ClangBuiltin<"__builtin_msa_sll_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_sll_w : GCCBuiltin<"__builtin_msa_sll_w">, +def int_mips_sll_w : ClangBuiltin<"__builtin_msa_sll_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_sll_d : GCCBuiltin<"__builtin_msa_sll_d">, +def int_mips_sll_d : ClangBuiltin<"__builtin_msa_sll_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_slli_b : GCCBuiltin<"__builtin_msa_slli_b">, +def int_mips_slli_b : ClangBuiltin<"__builtin_msa_slli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_slli_h : GCCBuiltin<"__builtin_msa_slli_h">, +def int_mips_slli_h : ClangBuiltin<"__builtin_msa_slli_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_slli_w : GCCBuiltin<"__builtin_msa_slli_w">, +def int_mips_slli_w : ClangBuiltin<"__builtin_msa_slli_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_slli_d : GCCBuiltin<"__builtin_msa_slli_d">, +def int_mips_slli_d : ClangBuiltin<"__builtin_msa_slli_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_splat_b : GCCBuiltin<"__builtin_msa_splat_b">, +def int_mips_splat_b : ClangBuiltin<"__builtin_msa_splat_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_splat_h : GCCBuiltin<"__builtin_msa_splat_h">, +def int_mips_splat_h : ClangBuiltin<"__builtin_msa_splat_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_splat_w : GCCBuiltin<"__builtin_msa_splat_w">, +def int_mips_splat_w : ClangBuiltin<"__builtin_msa_splat_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_splat_d : GCCBuiltin<"__builtin_msa_splat_d">, +def int_mips_splat_d : ClangBuiltin<"__builtin_msa_splat_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; -def int_mips_splati_b : GCCBuiltin<"__builtin_msa_splati_b">, +def int_mips_splati_b : ClangBuiltin<"__builtin_msa_splati_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_splati_h : GCCBuiltin<"__builtin_msa_splati_h">, +def int_mips_splati_h : ClangBuiltin<"__builtin_msa_splati_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_splati_w : GCCBuiltin<"__builtin_msa_splati_w">, +def int_mips_splati_w : ClangBuiltin<"__builtin_msa_splati_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_splati_d : GCCBuiltin<"__builtin_msa_splati_d">, +def int_mips_splati_d : ClangBuiltin<"__builtin_msa_splati_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_sra_b : GCCBuiltin<"__builtin_msa_sra_b">, +def int_mips_sra_b : ClangBuiltin<"__builtin_msa_sra_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_sra_h : GCCBuiltin<"__builtin_msa_sra_h">, +def int_mips_sra_h : ClangBuiltin<"__builtin_msa_sra_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_sra_w : GCCBuiltin<"__builtin_msa_sra_w">, +def int_mips_sra_w : ClangBuiltin<"__builtin_msa_sra_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_sra_d : GCCBuiltin<"__builtin_msa_sra_d">, +def int_mips_sra_d : ClangBuiltin<"__builtin_msa_sra_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_srai_b : GCCBuiltin<"__builtin_msa_srai_b">, +def int_mips_srai_b : ClangBuiltin<"__builtin_msa_srai_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srai_h : GCCBuiltin<"__builtin_msa_srai_h">, +def int_mips_srai_h : ClangBuiltin<"__builtin_msa_srai_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srai_w : GCCBuiltin<"__builtin_msa_srai_w">, +def int_mips_srai_w : ClangBuiltin<"__builtin_msa_srai_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srai_d : GCCBuiltin<"__builtin_msa_srai_d">, +def int_mips_srai_d : ClangBuiltin<"__builtin_msa_srai_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srar_b : GCCBuiltin<"__builtin_msa_srar_b">, +def int_mips_srar_b : ClangBuiltin<"__builtin_msa_srar_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_srar_h : GCCBuiltin<"__builtin_msa_srar_h">, +def int_mips_srar_h : ClangBuiltin<"__builtin_msa_srar_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_srar_w : GCCBuiltin<"__builtin_msa_srar_w">, +def int_mips_srar_w : ClangBuiltin<"__builtin_msa_srar_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_srar_d : GCCBuiltin<"__builtin_msa_srar_d">, +def int_mips_srar_d : ClangBuiltin<"__builtin_msa_srar_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_srari_b : GCCBuiltin<"__builtin_msa_srari_b">, +def int_mips_srari_b : ClangBuiltin<"__builtin_msa_srari_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srari_h : GCCBuiltin<"__builtin_msa_srari_h">, +def int_mips_srari_h : ClangBuiltin<"__builtin_msa_srari_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srari_w : GCCBuiltin<"__builtin_msa_srari_w">, +def int_mips_srari_w : ClangBuiltin<"__builtin_msa_srari_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srari_d : GCCBuiltin<"__builtin_msa_srari_d">, +def int_mips_srari_d : ClangBuiltin<"__builtin_msa_srari_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srl_b : GCCBuiltin<"__builtin_msa_srl_b">, +def int_mips_srl_b : ClangBuiltin<"__builtin_msa_srl_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_srl_h : GCCBuiltin<"__builtin_msa_srl_h">, +def int_mips_srl_h : ClangBuiltin<"__builtin_msa_srl_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_srl_w : GCCBuiltin<"__builtin_msa_srl_w">, +def int_mips_srl_w : ClangBuiltin<"__builtin_msa_srl_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_srl_d : GCCBuiltin<"__builtin_msa_srl_d">, +def int_mips_srl_d : ClangBuiltin<"__builtin_msa_srl_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_srli_b : GCCBuiltin<"__builtin_msa_srli_b">, +def int_mips_srli_b : ClangBuiltin<"__builtin_msa_srli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srli_h : GCCBuiltin<"__builtin_msa_srli_h">, +def int_mips_srli_h : ClangBuiltin<"__builtin_msa_srli_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srli_w : GCCBuiltin<"__builtin_msa_srli_w">, +def int_mips_srli_w : ClangBuiltin<"__builtin_msa_srli_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srli_d : GCCBuiltin<"__builtin_msa_srli_d">, +def int_mips_srli_d : ClangBuiltin<"__builtin_msa_srli_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srlr_b : GCCBuiltin<"__builtin_msa_srlr_b">, +def int_mips_srlr_b : ClangBuiltin<"__builtin_msa_srlr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_srlr_h : GCCBuiltin<"__builtin_msa_srlr_h">, +def int_mips_srlr_h : ClangBuiltin<"__builtin_msa_srlr_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_srlr_w : GCCBuiltin<"__builtin_msa_srlr_w">, +def int_mips_srlr_w : ClangBuiltin<"__builtin_msa_srlr_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_srlr_d : GCCBuiltin<"__builtin_msa_srlr_d">, +def int_mips_srlr_d : ClangBuiltin<"__builtin_msa_srlr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_srlri_b : GCCBuiltin<"__builtin_msa_srlri_b">, +def int_mips_srlri_b : ClangBuiltin<"__builtin_msa_srlri_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srlri_h : GCCBuiltin<"__builtin_msa_srlri_h">, +def int_mips_srlri_h : ClangBuiltin<"__builtin_msa_srlri_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srlri_w : GCCBuiltin<"__builtin_msa_srlri_w">, +def int_mips_srlri_w : ClangBuiltin<"__builtin_msa_srlri_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_srlri_d : GCCBuiltin<"__builtin_msa_srlri_d">, +def int_mips_srlri_d : ClangBuiltin<"__builtin_msa_srlri_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_st_b : GCCBuiltin<"__builtin_msa_st_b">, +def int_mips_st_b : ClangBuiltin<"__builtin_msa_st_b">, Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_st_h : GCCBuiltin<"__builtin_msa_st_h">, +def int_mips_st_h : ClangBuiltin<"__builtin_msa_st_h">, Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_st_w : GCCBuiltin<"__builtin_msa_st_w">, +def int_mips_st_w : ClangBuiltin<"__builtin_msa_st_w">, Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_st_d : GCCBuiltin<"__builtin_msa_st_d">, +def int_mips_st_d : ClangBuiltin<"__builtin_msa_st_d">, Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_str_d : GCCBuiltin<"__builtin_msa_str_d">, +def int_mips_str_d : ClangBuiltin<"__builtin_msa_str_d">, Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_str_w : GCCBuiltin<"__builtin_msa_str_w">, +def int_mips_str_w : ClangBuiltin<"__builtin_msa_str_w">, Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly]>; -def int_mips_subs_s_b : GCCBuiltin<"__builtin_msa_subs_s_b">, +def int_mips_subs_s_b : ClangBuiltin<"__builtin_msa_subs_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_subs_s_h : GCCBuiltin<"__builtin_msa_subs_s_h">, +def int_mips_subs_s_h : ClangBuiltin<"__builtin_msa_subs_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_subs_s_w : GCCBuiltin<"__builtin_msa_subs_s_w">, +def int_mips_subs_s_w : ClangBuiltin<"__builtin_msa_subs_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_subs_s_d : GCCBuiltin<"__builtin_msa_subs_s_d">, +def int_mips_subs_s_d : ClangBuiltin<"__builtin_msa_subs_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_subs_u_b : GCCBuiltin<"__builtin_msa_subs_u_b">, +def int_mips_subs_u_b : ClangBuiltin<"__builtin_msa_subs_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_subs_u_h : GCCBuiltin<"__builtin_msa_subs_u_h">, +def int_mips_subs_u_h : ClangBuiltin<"__builtin_msa_subs_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_subs_u_w : GCCBuiltin<"__builtin_msa_subs_u_w">, +def int_mips_subs_u_w : ClangBuiltin<"__builtin_msa_subs_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_subs_u_d : GCCBuiltin<"__builtin_msa_subs_u_d">, +def int_mips_subs_u_d : ClangBuiltin<"__builtin_msa_subs_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_subsus_u_b : GCCBuiltin<"__builtin_msa_subsus_u_b">, +def int_mips_subsus_u_b : ClangBuiltin<"__builtin_msa_subsus_u_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_subsus_u_h : GCCBuiltin<"__builtin_msa_subsus_u_h">, +def int_mips_subsus_u_h : ClangBuiltin<"__builtin_msa_subsus_u_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_subsus_u_w : GCCBuiltin<"__builtin_msa_subsus_u_w">, +def int_mips_subsus_u_w : ClangBuiltin<"__builtin_msa_subsus_u_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_subsus_u_d : GCCBuiltin<"__builtin_msa_subsus_u_d">, +def int_mips_subsus_u_d : ClangBuiltin<"__builtin_msa_subsus_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_subsuu_s_b : GCCBuiltin<"__builtin_msa_subsuu_s_b">, +def int_mips_subsuu_s_b : ClangBuiltin<"__builtin_msa_subsuu_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_subsuu_s_h : GCCBuiltin<"__builtin_msa_subsuu_s_h">, +def int_mips_subsuu_s_h : ClangBuiltin<"__builtin_msa_subsuu_s_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_subsuu_s_w : GCCBuiltin<"__builtin_msa_subsuu_s_w">, +def int_mips_subsuu_s_w : ClangBuiltin<"__builtin_msa_subsuu_s_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_subsuu_s_d : GCCBuiltin<"__builtin_msa_subsuu_s_d">, +def int_mips_subsuu_s_d : ClangBuiltin<"__builtin_msa_subsuu_s_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_subv_b : GCCBuiltin<"__builtin_msa_subv_b">, +def int_mips_subv_b : ClangBuiltin<"__builtin_msa_subv_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_subv_h : GCCBuiltin<"__builtin_msa_subv_h">, +def int_mips_subv_h : ClangBuiltin<"__builtin_msa_subv_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_subv_w : GCCBuiltin<"__builtin_msa_subv_w">, +def int_mips_subv_w : ClangBuiltin<"__builtin_msa_subv_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_subv_d : GCCBuiltin<"__builtin_msa_subv_d">, +def int_mips_subv_d : ClangBuiltin<"__builtin_msa_subv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_subvi_b : GCCBuiltin<"__builtin_msa_subvi_b">, +def int_mips_subvi_b : ClangBuiltin<"__builtin_msa_subvi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_subvi_h : GCCBuiltin<"__builtin_msa_subvi_h">, +def int_mips_subvi_h : ClangBuiltin<"__builtin_msa_subvi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_subvi_w : GCCBuiltin<"__builtin_msa_subvi_w">, +def int_mips_subvi_w : ClangBuiltin<"__builtin_msa_subvi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_subvi_d : GCCBuiltin<"__builtin_msa_subvi_d">, +def int_mips_subvi_d : ClangBuiltin<"__builtin_msa_subvi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; -def int_mips_vshf_b : GCCBuiltin<"__builtin_msa_vshf_b">, +def int_mips_vshf_b : ClangBuiltin<"__builtin_msa_vshf_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_vshf_h : GCCBuiltin<"__builtin_msa_vshf_h">, +def int_mips_vshf_h : ClangBuiltin<"__builtin_msa_vshf_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; -def int_mips_vshf_w : GCCBuiltin<"__builtin_msa_vshf_w">, +def int_mips_vshf_w : ClangBuiltin<"__builtin_msa_vshf_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_mips_vshf_d : GCCBuiltin<"__builtin_msa_vshf_d">, +def int_mips_vshf_d : ClangBuiltin<"__builtin_msa_vshf_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; -def int_mips_xor_v : GCCBuiltin<"__builtin_msa_xor_v">, +def int_mips_xor_v : ClangBuiltin<"__builtin_msa_xor_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_mips_xori_b : GCCBuiltin<"__builtin_msa_xori_b">, +def int_mips_xori_b : ClangBuiltin<"__builtin_msa_xori_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 41b28db56c75..9c3813128364 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -556,95 +556,124 @@ class SHFL_INFO<bit sync, string mode, string type, bit return_pred> { } let TargetPrefix = "nvvm" in { - def int_nvvm_prmt : GCCBuiltin<"__nvvm_prmt">, + def int_nvvm_prmt : ClangBuiltin<"__nvvm_prmt">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, Commutative]>; + [IntrNoMem, IntrSpeculatable]>; // // Min Max // - def int_nvvm_fmin_f : GCCBuiltin<"__nvvm_fmin_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_fmin_ftz_f : GCCBuiltin<"__nvvm_fmin_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], + foreach operation = ["min", "max"] in { + def int_nvvm_f # operation # _d : + ClangBuiltin<!strconcat("__nvvm_f", operation, "_d")>, + DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_fmax_f : GCCBuiltin<"__nvvm_fmax_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty] - , [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_fmax_ftz_f : GCCBuiltin<"__nvvm_fmax_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable, Commutative]>; + foreach variant = ["_f", "_ftz_f", "_nan_f", "_ftz_nan_f", + "_xorsign_abs_f", "_ftz_xorsign_abs_f", "_nan_xorsign_abs_f", + "_ftz_nan_xorsign_abs_f"] in { + def int_nvvm_f # operation # variant : + ClangBuiltin<!strconcat("__nvvm_f", operation, variant)>, + DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable, Commutative]>; + } - def int_nvvm_fmin_d : GCCBuiltin<"__nvvm_fmin_d">, - DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], - [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_fmax_d : GCCBuiltin<"__nvvm_fmax_d">, - DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], - [IntrNoMem, IntrSpeculatable, Commutative]>; + foreach variant = ["_f16", "_ftz_f16", "_nan_f16", "_ftz_nan_f16", + "_xorsign_abs_f16", "_ftz_xorsign_abs_f16", "_nan_xorsign_abs_f16", + "_ftz_nan_xorsign_abs_f16"] in { + def int_nvvm_f # operation # variant : + ClangBuiltin<!strconcat("__nvvm_f", operation, variant)>, + DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty, llvm_half_ty], + [IntrNoMem, IntrSpeculatable, Commutative]>; + } + + foreach variant = ["_f16x2", "_ftz_f16x2", "_nan_f16x2", + "_ftz_nan_f16x2", "_xorsign_abs_f16x2", "_ftz_xorsign_abs_f16x2", + "_nan_xorsign_abs_f16x2", "_ftz_nan_xorsign_abs_f16x2"] in { + def int_nvvm_f # operation # variant : + ClangBuiltin<!strconcat("__nvvm_f", operation, variant)>, + DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty, llvm_v2f16_ty], + [IntrNoMem, IntrSpeculatable, Commutative]>; + } + + foreach variant = ["_bf16", "_nan_bf16", "_xorsign_abs_bf16", + "_nan_xorsign_abs_bf16"] in { + def int_nvvm_f # operation # variant : + ClangBuiltin<!strconcat("__nvvm_f", operation, variant)>, + DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], + [IntrNoMem, IntrSpeculatable, Commutative]>; + } + + foreach variant = ["_bf16x2", "_nan_bf16x2", "_xorsign_abs_bf16x2", + "_nan_xorsign_abs_bf16x2"] in { + def int_nvvm_f # operation # variant : + ClangBuiltin<!strconcat("__nvvm_f", operation, variant)>, + DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, Commutative]>; + } + } // // Multiplication // - def int_nvvm_mulhi_i : GCCBuiltin<"__nvvm_mulhi_i">, + def int_nvvm_mulhi_i : ClangBuiltin<"__nvvm_mulhi_i">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mulhi_ui : GCCBuiltin<"__nvvm_mulhi_ui">, + def int_nvvm_mulhi_ui : ClangBuiltin<"__nvvm_mulhi_ui">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mulhi_ll : GCCBuiltin<"__nvvm_mulhi_ll">, + def int_nvvm_mulhi_ll : ClangBuiltin<"__nvvm_mulhi_ll">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mulhi_ull : GCCBuiltin<"__nvvm_mulhi_ull">, + def int_nvvm_mulhi_ull : ClangBuiltin<"__nvvm_mulhi_ull">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rn_ftz_f : GCCBuiltin<"__nvvm_mul_rn_ftz_f">, + def int_nvvm_mul_rn_ftz_f : ClangBuiltin<"__nvvm_mul_rn_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rn_f : GCCBuiltin<"__nvvm_mul_rn_f">, + def int_nvvm_mul_rn_f : ClangBuiltin<"__nvvm_mul_rn_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rz_ftz_f : GCCBuiltin<"__nvvm_mul_rz_ftz_f">, + def int_nvvm_mul_rz_ftz_f : ClangBuiltin<"__nvvm_mul_rz_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rz_f : GCCBuiltin<"__nvvm_mul_rz_f">, + def int_nvvm_mul_rz_f : ClangBuiltin<"__nvvm_mul_rz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rm_ftz_f : GCCBuiltin<"__nvvm_mul_rm_ftz_f">, + def int_nvvm_mul_rm_ftz_f : ClangBuiltin<"__nvvm_mul_rm_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rm_f : GCCBuiltin<"__nvvm_mul_rm_f">, + def int_nvvm_mul_rm_f : ClangBuiltin<"__nvvm_mul_rm_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rp_ftz_f : GCCBuiltin<"__nvvm_mul_rp_ftz_f">, + def int_nvvm_mul_rp_ftz_f : ClangBuiltin<"__nvvm_mul_rp_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rp_f : GCCBuiltin<"__nvvm_mul_rp_f">, + def int_nvvm_mul_rp_f : ClangBuiltin<"__nvvm_mul_rp_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rn_d : GCCBuiltin<"__nvvm_mul_rn_d">, + def int_nvvm_mul_rn_d : ClangBuiltin<"__nvvm_mul_rn_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rz_d : GCCBuiltin<"__nvvm_mul_rz_d">, + def int_nvvm_mul_rz_d : ClangBuiltin<"__nvvm_mul_rz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rm_d : GCCBuiltin<"__nvvm_mul_rm_d">, + def int_nvvm_mul_rm_d : ClangBuiltin<"__nvvm_mul_rm_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul_rp_d : GCCBuiltin<"__nvvm_mul_rp_d">, + def int_nvvm_mul_rp_d : ClangBuiltin<"__nvvm_mul_rp_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul24_i : GCCBuiltin<"__nvvm_mul24_i">, + def int_nvvm_mul24_i : ClangBuiltin<"__nvvm_mul24_i">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_mul24_ui : GCCBuiltin<"__nvvm_mul24_ui">, + def int_nvvm_mul24_ui : ClangBuiltin<"__nvvm_mul24_ui">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; @@ -652,51 +681,51 @@ let TargetPrefix = "nvvm" in { // Div // - def int_nvvm_div_approx_ftz_f : GCCBuiltin<"__nvvm_div_approx_ftz_f">, + def int_nvvm_div_approx_ftz_f : ClangBuiltin<"__nvvm_div_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_approx_f : GCCBuiltin<"__nvvm_div_approx_f">, + def int_nvvm_div_approx_f : ClangBuiltin<"__nvvm_div_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rn_ftz_f : GCCBuiltin<"__nvvm_div_rn_ftz_f">, + def int_nvvm_div_rn_ftz_f : ClangBuiltin<"__nvvm_div_rn_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rn_f : GCCBuiltin<"__nvvm_div_rn_f">, + def int_nvvm_div_rn_f : ClangBuiltin<"__nvvm_div_rn_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rz_ftz_f : GCCBuiltin<"__nvvm_div_rz_ftz_f">, + def int_nvvm_div_rz_ftz_f : ClangBuiltin<"__nvvm_div_rz_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rz_f : GCCBuiltin<"__nvvm_div_rz_f">, + def int_nvvm_div_rz_f : ClangBuiltin<"__nvvm_div_rz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rm_ftz_f : GCCBuiltin<"__nvvm_div_rm_ftz_f">, + def int_nvvm_div_rm_ftz_f : ClangBuiltin<"__nvvm_div_rm_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rm_f : GCCBuiltin<"__nvvm_div_rm_f">, + def int_nvvm_div_rm_f : ClangBuiltin<"__nvvm_div_rm_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rp_ftz_f : GCCBuiltin<"__nvvm_div_rp_ftz_f">, + def int_nvvm_div_rp_ftz_f : ClangBuiltin<"__nvvm_div_rp_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rp_f : GCCBuiltin<"__nvvm_div_rp_f">, + def int_nvvm_div_rp_f : ClangBuiltin<"__nvvm_div_rp_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_div_rn_d : GCCBuiltin<"__nvvm_div_rn_d">, + def int_nvvm_div_rn_d : ClangBuiltin<"__nvvm_div_rn_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_nvvm_div_rz_d : GCCBuiltin<"__nvvm_div_rz_d">, + def int_nvvm_div_rz_d : ClangBuiltin<"__nvvm_div_rz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_nvvm_div_rm_d : GCCBuiltin<"__nvvm_div_rm_d">, + def int_nvvm_div_rm_d : ClangBuiltin<"__nvvm_div_rm_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_nvvm_div_rp_d : GCCBuiltin<"__nvvm_div_rp_d">, + def int_nvvm_div_rp_d : ClangBuiltin<"__nvvm_div_rp_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; @@ -704,10 +733,10 @@ let TargetPrefix = "nvvm" in { // Sad // - def int_nvvm_sad_i : GCCBuiltin<"__nvvm_sad_i">, + def int_nvvm_sad_i : ClangBuiltin<"__nvvm_sad_i">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; - def int_nvvm_sad_ui : GCCBuiltin<"__nvvm_sad_ui">, + def int_nvvm_sad_ui : ClangBuiltin<"__nvvm_sad_ui">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, Commutative]>; @@ -715,264 +744,286 @@ let TargetPrefix = "nvvm" in { // Floor Ceil // - def int_nvvm_floor_ftz_f : GCCBuiltin<"__nvvm_floor_ftz_f">, + def int_nvvm_floor_ftz_f : ClangBuiltin<"__nvvm_floor_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_floor_f : GCCBuiltin<"__nvvm_floor_f">, + def int_nvvm_floor_f : ClangBuiltin<"__nvvm_floor_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_floor_d : GCCBuiltin<"__nvvm_floor_d">, + def int_nvvm_floor_d : ClangBuiltin<"__nvvm_floor_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ceil_ftz_f : GCCBuiltin<"__nvvm_ceil_ftz_f">, + def int_nvvm_ceil_ftz_f : ClangBuiltin<"__nvvm_ceil_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ceil_f : GCCBuiltin<"__nvvm_ceil_f">, + def int_nvvm_ceil_f : ClangBuiltin<"__nvvm_ceil_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ceil_d : GCCBuiltin<"__nvvm_ceil_d">, + def int_nvvm_ceil_d : ClangBuiltin<"__nvvm_ceil_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // // Abs // - def int_nvvm_fabs_ftz_f : GCCBuiltin<"__nvvm_fabs_ftz_f">, + def int_nvvm_fabs_ftz_f : ClangBuiltin<"__nvvm_fabs_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fabs_f : GCCBuiltin<"__nvvm_fabs_f">, + def int_nvvm_fabs_f : ClangBuiltin<"__nvvm_fabs_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fabs_d : GCCBuiltin<"__nvvm_fabs_d">, + def int_nvvm_fabs_d : ClangBuiltin<"__nvvm_fabs_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // +// Abs, Neg bf16, bf16x2 +// + + foreach unary = ["abs", "neg"] in { + def int_nvvm_ # unary # _bf16 : + ClangBuiltin<!strconcat("__nvvm_", unary, "_bf16")>, + DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_nvvm_ # unary # _bf16x2 : + ClangBuiltin<!strconcat("__nvvm_", unary, "_bf16x2")>, + DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; + } + +// // Round // - def int_nvvm_round_ftz_f : GCCBuiltin<"__nvvm_round_ftz_f">, + def int_nvvm_round_ftz_f : ClangBuiltin<"__nvvm_round_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_round_f : GCCBuiltin<"__nvvm_round_f">, + def int_nvvm_round_f : ClangBuiltin<"__nvvm_round_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_round_d : GCCBuiltin<"__nvvm_round_d">, + def int_nvvm_round_d : ClangBuiltin<"__nvvm_round_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // // Trunc // - def int_nvvm_trunc_ftz_f : GCCBuiltin<"__nvvm_trunc_ftz_f">, + def int_nvvm_trunc_ftz_f : ClangBuiltin<"__nvvm_trunc_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_trunc_f : GCCBuiltin<"__nvvm_trunc_f">, + def int_nvvm_trunc_f : ClangBuiltin<"__nvvm_trunc_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_trunc_d : GCCBuiltin<"__nvvm_trunc_d">, + def int_nvvm_trunc_d : ClangBuiltin<"__nvvm_trunc_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // // Saturate // - def int_nvvm_saturate_ftz_f : GCCBuiltin<"__nvvm_saturate_ftz_f">, + def int_nvvm_saturate_ftz_f : ClangBuiltin<"__nvvm_saturate_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_saturate_f : GCCBuiltin<"__nvvm_saturate_f">, + def int_nvvm_saturate_f : ClangBuiltin<"__nvvm_saturate_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_saturate_d : GCCBuiltin<"__nvvm_saturate_d">, + def int_nvvm_saturate_d : ClangBuiltin<"__nvvm_saturate_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // // Exp2 Log2 // - def int_nvvm_ex2_approx_ftz_f : GCCBuiltin<"__nvvm_ex2_approx_ftz_f">, + def int_nvvm_ex2_approx_ftz_f : ClangBuiltin<"__nvvm_ex2_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ex2_approx_f : GCCBuiltin<"__nvvm_ex2_approx_f">, + def int_nvvm_ex2_approx_f : ClangBuiltin<"__nvvm_ex2_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ex2_approx_d : GCCBuiltin<"__nvvm_ex2_approx_d">, + def int_nvvm_ex2_approx_d : ClangBuiltin<"__nvvm_ex2_approx_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; + def int_nvvm_ex2_approx_f16 : ClangBuiltin<"__nvvm_ex2_approx_f16">, + DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty], [IntrNoMem]>; + def int_nvvm_ex2_approx_f16x2 : ClangBuiltin<"__nvvm_ex2_approx_f16x2">, + DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty], [IntrNoMem]>; - def int_nvvm_lg2_approx_ftz_f : GCCBuiltin<"__nvvm_lg2_approx_ftz_f">, + def int_nvvm_lg2_approx_ftz_f : ClangBuiltin<"__nvvm_lg2_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_lg2_approx_f : GCCBuiltin<"__nvvm_lg2_approx_f">, + def int_nvvm_lg2_approx_f : ClangBuiltin<"__nvvm_lg2_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_lg2_approx_d : GCCBuiltin<"__nvvm_lg2_approx_d">, + def int_nvvm_lg2_approx_d : ClangBuiltin<"__nvvm_lg2_approx_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; // // Sin Cos // - def int_nvvm_sin_approx_ftz_f : GCCBuiltin<"__nvvm_sin_approx_ftz_f">, + def int_nvvm_sin_approx_ftz_f : ClangBuiltin<"__nvvm_sin_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sin_approx_f : GCCBuiltin<"__nvvm_sin_approx_f">, + def int_nvvm_sin_approx_f : ClangBuiltin<"__nvvm_sin_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_cos_approx_ftz_f : GCCBuiltin<"__nvvm_cos_approx_ftz_f">, + def int_nvvm_cos_approx_ftz_f : ClangBuiltin<"__nvvm_cos_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_cos_approx_f : GCCBuiltin<"__nvvm_cos_approx_f">, + def int_nvvm_cos_approx_f : ClangBuiltin<"__nvvm_cos_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; // // Fma // - def int_nvvm_fma_rn_ftz_f : GCCBuiltin<"__nvvm_fma_rn_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rn_f : GCCBuiltin<"__nvvm_fma_rn_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rz_ftz_f : GCCBuiltin<"__nvvm_fma_rz_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rz_f : GCCBuiltin<"__nvvm_fma_rz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rm_ftz_f : GCCBuiltin<"__nvvm_fma_rm_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rm_f : GCCBuiltin<"__nvvm_fma_rm_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rp_ftz_f : GCCBuiltin<"__nvvm_fma_rp_ftz_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rp_f : GCCBuiltin<"__nvvm_fma_rp_f">, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], + foreach variant = ["_rn_f16", "_rn_ftz_f16", "_rn_sat_f16", + "_rn_ftz_sat_f16", "_rn_relu_f16", "_rn_ftz_relu_f16"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, + DefaultAttrsIntrinsic<[llvm_half_ty], + [llvm_half_ty, llvm_half_ty, llvm_half_ty], + [IntrNoMem, IntrSpeculatable]>; + } + + foreach variant = ["_rn_f16x2", "_rn_ftz_f16x2", "_rn_sat_f16x2", + "_rn_ftz_sat_f16x2", "_rn_relu_f16x2", "_rn_ftz_relu_f16x2"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, + DefaultAttrsIntrinsic<[llvm_v2f16_ty], + [llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty], [IntrNoMem, IntrSpeculatable]>; + } - def int_nvvm_fma_rn_d : GCCBuiltin<"__nvvm_fma_rn_d">, - DefaultAttrsIntrinsic<[llvm_double_ty], - [llvm_double_ty, llvm_double_ty, llvm_double_ty], + foreach variant = ["_rn_bf16", "_rn_relu_bf16"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, + DefaultAttrsIntrinsic<[llvm_i16_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rz_d : GCCBuiltin<"__nvvm_fma_rz_d">, - DefaultAttrsIntrinsic<[llvm_double_ty], - [llvm_double_ty, llvm_double_ty, llvm_double_ty], + } + + foreach variant = ["_rn_bf16x2", "_rn_relu_bf16x2"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, + DefaultAttrsIntrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rm_d : GCCBuiltin<"__nvvm_fma_rm_d">, - DefaultAttrsIntrinsic<[llvm_double_ty], - [llvm_double_ty, llvm_double_ty, llvm_double_ty], + } + + foreach variant = ["_rn_ftz_f", "_rn_f", "_rz_ftz_f", "_rz_f", "_rm_ftz_f", + "_rm_f", "_rp_ftz_f", "_rp_f"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, + DefaultAttrsIntrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_fma_rp_d : GCCBuiltin<"__nvvm_fma_rp_d">, + } + + foreach variant = ["_rn_d", "_rz_d", "_rm_d", "_rp_d"] in { + def int_nvvm_fma # variant : ClangBuiltin<!strconcat("__nvvm_fma", variant)>, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; + } // // Rcp // - def int_nvvm_rcp_rn_ftz_f : GCCBuiltin<"__nvvm_rcp_rn_ftz_f">, + def int_nvvm_rcp_rn_ftz_f : ClangBuiltin<"__nvvm_rcp_rn_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rn_f : GCCBuiltin<"__nvvm_rcp_rn_f">, + def int_nvvm_rcp_rn_f : ClangBuiltin<"__nvvm_rcp_rn_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rz_ftz_f : GCCBuiltin<"__nvvm_rcp_rz_ftz_f">, + def int_nvvm_rcp_rz_ftz_f : ClangBuiltin<"__nvvm_rcp_rz_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rz_f : GCCBuiltin<"__nvvm_rcp_rz_f">, + def int_nvvm_rcp_rz_f : ClangBuiltin<"__nvvm_rcp_rz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rm_ftz_f : GCCBuiltin<"__nvvm_rcp_rm_ftz_f">, + def int_nvvm_rcp_rm_ftz_f : ClangBuiltin<"__nvvm_rcp_rm_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rm_f : GCCBuiltin<"__nvvm_rcp_rm_f">, + def int_nvvm_rcp_rm_f : ClangBuiltin<"__nvvm_rcp_rm_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rp_ftz_f : GCCBuiltin<"__nvvm_rcp_rp_ftz_f">, + def int_nvvm_rcp_rp_ftz_f : ClangBuiltin<"__nvvm_rcp_rp_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rp_f : GCCBuiltin<"__nvvm_rcp_rp_f">, + def int_nvvm_rcp_rp_f : ClangBuiltin<"__nvvm_rcp_rp_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rcp_rn_d : GCCBuiltin<"__nvvm_rcp_rn_d">, + def int_nvvm_rcp_rn_d : ClangBuiltin<"__nvvm_rcp_rn_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_rcp_rz_d : GCCBuiltin<"__nvvm_rcp_rz_d">, + def int_nvvm_rcp_rz_d : ClangBuiltin<"__nvvm_rcp_rz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_rcp_rm_d : GCCBuiltin<"__nvvm_rcp_rm_d">, + def int_nvvm_rcp_rm_d : ClangBuiltin<"__nvvm_rcp_rm_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_rcp_rp_d : GCCBuiltin<"__nvvm_rcp_rp_d">, + def int_nvvm_rcp_rp_d : ClangBuiltin<"__nvvm_rcp_rp_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_rcp_approx_ftz_d : GCCBuiltin<"__nvvm_rcp_approx_ftz_d">, + def int_nvvm_rcp_approx_ftz_f : ClangBuiltin<"__nvvm_rcp_approx_ftz_f">, + DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; + def int_nvvm_rcp_approx_ftz_d : ClangBuiltin<"__nvvm_rcp_approx_ftz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; // // Sqrt // - def int_nvvm_sqrt_f : GCCBuiltin<"__nvvm_sqrt_f">, + def int_nvvm_sqrt_f : ClangBuiltin<"__nvvm_sqrt_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rn_ftz_f : GCCBuiltin<"__nvvm_sqrt_rn_ftz_f">, + def int_nvvm_sqrt_rn_ftz_f : ClangBuiltin<"__nvvm_sqrt_rn_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rn_f : GCCBuiltin<"__nvvm_sqrt_rn_f">, + def int_nvvm_sqrt_rn_f : ClangBuiltin<"__nvvm_sqrt_rn_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rz_ftz_f : GCCBuiltin<"__nvvm_sqrt_rz_ftz_f">, + def int_nvvm_sqrt_rz_ftz_f : ClangBuiltin<"__nvvm_sqrt_rz_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rz_f : GCCBuiltin<"__nvvm_sqrt_rz_f">, + def int_nvvm_sqrt_rz_f : ClangBuiltin<"__nvvm_sqrt_rz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rm_ftz_f : GCCBuiltin<"__nvvm_sqrt_rm_ftz_f">, + def int_nvvm_sqrt_rm_ftz_f : ClangBuiltin<"__nvvm_sqrt_rm_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rm_f : GCCBuiltin<"__nvvm_sqrt_rm_f">, + def int_nvvm_sqrt_rm_f : ClangBuiltin<"__nvvm_sqrt_rm_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rp_ftz_f : GCCBuiltin<"__nvvm_sqrt_rp_ftz_f">, + def int_nvvm_sqrt_rp_ftz_f : ClangBuiltin<"__nvvm_sqrt_rp_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rp_f : GCCBuiltin<"__nvvm_sqrt_rp_f">, + def int_nvvm_sqrt_rp_f : ClangBuiltin<"__nvvm_sqrt_rp_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_approx_ftz_f : GCCBuiltin<"__nvvm_sqrt_approx_ftz_f">, + def int_nvvm_sqrt_approx_ftz_f : ClangBuiltin<"__nvvm_sqrt_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_approx_f : GCCBuiltin<"__nvvm_sqrt_approx_f">, + def int_nvvm_sqrt_approx_f : ClangBuiltin<"__nvvm_sqrt_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rn_d : GCCBuiltin<"__nvvm_sqrt_rn_d">, + def int_nvvm_sqrt_rn_d : ClangBuiltin<"__nvvm_sqrt_rn_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rz_d : GCCBuiltin<"__nvvm_sqrt_rz_d">, + def int_nvvm_sqrt_rz_d : ClangBuiltin<"__nvvm_sqrt_rz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rm_d : GCCBuiltin<"__nvvm_sqrt_rm_d">, + def int_nvvm_sqrt_rm_d : ClangBuiltin<"__nvvm_sqrt_rm_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_nvvm_sqrt_rp_d : GCCBuiltin<"__nvvm_sqrt_rp_d">, + def int_nvvm_sqrt_rp_d : ClangBuiltin<"__nvvm_sqrt_rp_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; // // Rsqrt // - def int_nvvm_rsqrt_approx_ftz_f : GCCBuiltin<"__nvvm_rsqrt_approx_ftz_f">, + def int_nvvm_rsqrt_approx_ftz_f : ClangBuiltin<"__nvvm_rsqrt_approx_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rsqrt_approx_f : GCCBuiltin<"__nvvm_rsqrt_approx_f">, + def int_nvvm_rsqrt_approx_f : ClangBuiltin<"__nvvm_rsqrt_approx_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_rsqrt_approx_d : GCCBuiltin<"__nvvm_rsqrt_approx_d">, + def int_nvvm_rsqrt_approx_d : ClangBuiltin<"__nvvm_rsqrt_approx_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; // // Add // - def int_nvvm_add_rn_ftz_f : GCCBuiltin<"__nvvm_add_rn_ftz_f">, + def int_nvvm_add_rn_ftz_f : ClangBuiltin<"__nvvm_add_rn_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rn_f : GCCBuiltin<"__nvvm_add_rn_f">, + def int_nvvm_add_rn_f : ClangBuiltin<"__nvvm_add_rn_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rz_ftz_f : GCCBuiltin<"__nvvm_add_rz_ftz_f">, + def int_nvvm_add_rz_ftz_f : ClangBuiltin<"__nvvm_add_rz_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rz_f : GCCBuiltin<"__nvvm_add_rz_f">, + def int_nvvm_add_rz_f : ClangBuiltin<"__nvvm_add_rz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rm_ftz_f : GCCBuiltin<"__nvvm_add_rm_ftz_f">, + def int_nvvm_add_rm_ftz_f : ClangBuiltin<"__nvvm_add_rm_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rm_f : GCCBuiltin<"__nvvm_add_rm_f">, + def int_nvvm_add_rm_f : ClangBuiltin<"__nvvm_add_rm_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rp_ftz_f : GCCBuiltin<"__nvvm_add_rp_ftz_f">, + def int_nvvm_add_rp_ftz_f : ClangBuiltin<"__nvvm_add_rp_ftz_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rp_f : GCCBuiltin<"__nvvm_add_rp_f">, + def int_nvvm_add_rp_f : ClangBuiltin<"__nvvm_add_rp_f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rn_d : GCCBuiltin<"__nvvm_add_rn_d">, + def int_nvvm_add_rn_d : ClangBuiltin<"__nvvm_add_rn_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rz_d : GCCBuiltin<"__nvvm_add_rz_d">, + def int_nvvm_add_rz_d : ClangBuiltin<"__nvvm_add_rz_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rm_d : GCCBuiltin<"__nvvm_add_rm_d">, + def int_nvvm_add_rm_d : ClangBuiltin<"__nvvm_add_rm_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_add_rp_d : GCCBuiltin<"__nvvm_add_rp_d">, + def int_nvvm_add_rp_d : ClangBuiltin<"__nvvm_add_rp_d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; @@ -980,278 +1031,278 @@ let TargetPrefix = "nvvm" in { // Convert // - def int_nvvm_d2f_rn_ftz : GCCBuiltin<"__nvvm_d2f_rn_ftz">, + def int_nvvm_d2f_rn_ftz : ClangBuiltin<"__nvvm_d2f_rn_ftz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rn : GCCBuiltin<"__nvvm_d2f_rn">, + def int_nvvm_d2f_rn : ClangBuiltin<"__nvvm_d2f_rn">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rz_ftz : GCCBuiltin<"__nvvm_d2f_rz_ftz">, + def int_nvvm_d2f_rz_ftz : ClangBuiltin<"__nvvm_d2f_rz_ftz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rz : GCCBuiltin<"__nvvm_d2f_rz">, + def int_nvvm_d2f_rz : ClangBuiltin<"__nvvm_d2f_rz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rm_ftz : GCCBuiltin<"__nvvm_d2f_rm_ftz">, + def int_nvvm_d2f_rm_ftz : ClangBuiltin<"__nvvm_d2f_rm_ftz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rm : GCCBuiltin<"__nvvm_d2f_rm">, + def int_nvvm_d2f_rm : ClangBuiltin<"__nvvm_d2f_rm">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rp_ftz : GCCBuiltin<"__nvvm_d2f_rp_ftz">, + def int_nvvm_d2f_rp_ftz : ClangBuiltin<"__nvvm_d2f_rp_ftz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2f_rp : GCCBuiltin<"__nvvm_d2f_rp">, + def int_nvvm_d2f_rp : ClangBuiltin<"__nvvm_d2f_rp">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2i_rn : GCCBuiltin<"__nvvm_d2i_rn">, + def int_nvvm_d2i_rn : ClangBuiltin<"__nvvm_d2i_rn">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2i_rz : GCCBuiltin<"__nvvm_d2i_rz">, + def int_nvvm_d2i_rz : ClangBuiltin<"__nvvm_d2i_rz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2i_rm : GCCBuiltin<"__nvvm_d2i_rm">, + def int_nvvm_d2i_rm : ClangBuiltin<"__nvvm_d2i_rm">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2i_rp : GCCBuiltin<"__nvvm_d2i_rp">, + def int_nvvm_d2i_rp : ClangBuiltin<"__nvvm_d2i_rp">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ui_rn : GCCBuiltin<"__nvvm_d2ui_rn">, + def int_nvvm_d2ui_rn : ClangBuiltin<"__nvvm_d2ui_rn">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ui_rz : GCCBuiltin<"__nvvm_d2ui_rz">, + def int_nvvm_d2ui_rz : ClangBuiltin<"__nvvm_d2ui_rz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ui_rm : GCCBuiltin<"__nvvm_d2ui_rm">, + def int_nvvm_d2ui_rm : ClangBuiltin<"__nvvm_d2ui_rm">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ui_rp : GCCBuiltin<"__nvvm_d2ui_rp">, + def int_nvvm_d2ui_rp : ClangBuiltin<"__nvvm_d2ui_rp">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2d_rn : GCCBuiltin<"__nvvm_i2d_rn">, + def int_nvvm_i2d_rn : ClangBuiltin<"__nvvm_i2d_rn">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2d_rz : GCCBuiltin<"__nvvm_i2d_rz">, + def int_nvvm_i2d_rz : ClangBuiltin<"__nvvm_i2d_rz">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2d_rm : GCCBuiltin<"__nvvm_i2d_rm">, + def int_nvvm_i2d_rm : ClangBuiltin<"__nvvm_i2d_rm">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2d_rp : GCCBuiltin<"__nvvm_i2d_rp">, + def int_nvvm_i2d_rp : ClangBuiltin<"__nvvm_i2d_rp">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2d_rn : GCCBuiltin<"__nvvm_ui2d_rn">, + def int_nvvm_ui2d_rn : ClangBuiltin<"__nvvm_ui2d_rn">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2d_rz : GCCBuiltin<"__nvvm_ui2d_rz">, + def int_nvvm_ui2d_rz : ClangBuiltin<"__nvvm_ui2d_rz">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2d_rm : GCCBuiltin<"__nvvm_ui2d_rm">, + def int_nvvm_ui2d_rm : ClangBuiltin<"__nvvm_ui2d_rm">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2d_rp : GCCBuiltin<"__nvvm_ui2d_rp">, + def int_nvvm_ui2d_rp : ClangBuiltin<"__nvvm_ui2d_rp">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rn_ftz : GCCBuiltin<"__nvvm_f2i_rn_ftz">, + def int_nvvm_f2i_rn_ftz : ClangBuiltin<"__nvvm_f2i_rn_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rn : GCCBuiltin<"__nvvm_f2i_rn">, + def int_nvvm_f2i_rn : ClangBuiltin<"__nvvm_f2i_rn">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rz_ftz : GCCBuiltin<"__nvvm_f2i_rz_ftz">, + def int_nvvm_f2i_rz_ftz : ClangBuiltin<"__nvvm_f2i_rz_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rz : GCCBuiltin<"__nvvm_f2i_rz">, + def int_nvvm_f2i_rz : ClangBuiltin<"__nvvm_f2i_rz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rm_ftz : GCCBuiltin<"__nvvm_f2i_rm_ftz">, + def int_nvvm_f2i_rm_ftz : ClangBuiltin<"__nvvm_f2i_rm_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rm : GCCBuiltin<"__nvvm_f2i_rm">, + def int_nvvm_f2i_rm : ClangBuiltin<"__nvvm_f2i_rm">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rp_ftz : GCCBuiltin<"__nvvm_f2i_rp_ftz">, + def int_nvvm_f2i_rp_ftz : ClangBuiltin<"__nvvm_f2i_rp_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2i_rp : GCCBuiltin<"__nvvm_f2i_rp">, + def int_nvvm_f2i_rp : ClangBuiltin<"__nvvm_f2i_rp">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rn_ftz : GCCBuiltin<"__nvvm_f2ui_rn_ftz">, + def int_nvvm_f2ui_rn_ftz : ClangBuiltin<"__nvvm_f2ui_rn_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rn : GCCBuiltin<"__nvvm_f2ui_rn">, + def int_nvvm_f2ui_rn : ClangBuiltin<"__nvvm_f2ui_rn">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rz_ftz : GCCBuiltin<"__nvvm_f2ui_rz_ftz">, + def int_nvvm_f2ui_rz_ftz : ClangBuiltin<"__nvvm_f2ui_rz_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rz : GCCBuiltin<"__nvvm_f2ui_rz">, + def int_nvvm_f2ui_rz : ClangBuiltin<"__nvvm_f2ui_rz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rm_ftz : GCCBuiltin<"__nvvm_f2ui_rm_ftz">, + def int_nvvm_f2ui_rm_ftz : ClangBuiltin<"__nvvm_f2ui_rm_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rm : GCCBuiltin<"__nvvm_f2ui_rm">, + def int_nvvm_f2ui_rm : ClangBuiltin<"__nvvm_f2ui_rm">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rp_ftz : GCCBuiltin<"__nvvm_f2ui_rp_ftz">, + def int_nvvm_f2ui_rp_ftz : ClangBuiltin<"__nvvm_f2ui_rp_ftz">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ui_rp : GCCBuiltin<"__nvvm_f2ui_rp">, + def int_nvvm_f2ui_rp : ClangBuiltin<"__nvvm_f2ui_rp">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2f_rn : GCCBuiltin<"__nvvm_i2f_rn">, + def int_nvvm_i2f_rn : ClangBuiltin<"__nvvm_i2f_rn">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2f_rz : GCCBuiltin<"__nvvm_i2f_rz">, + def int_nvvm_i2f_rz : ClangBuiltin<"__nvvm_i2f_rz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2f_rm : GCCBuiltin<"__nvvm_i2f_rm">, + def int_nvvm_i2f_rm : ClangBuiltin<"__nvvm_i2f_rm">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_i2f_rp : GCCBuiltin<"__nvvm_i2f_rp">, + def int_nvvm_i2f_rp : ClangBuiltin<"__nvvm_i2f_rp">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2f_rn : GCCBuiltin<"__nvvm_ui2f_rn">, + def int_nvvm_ui2f_rn : ClangBuiltin<"__nvvm_ui2f_rn">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2f_rz : GCCBuiltin<"__nvvm_ui2f_rz">, + def int_nvvm_ui2f_rz : ClangBuiltin<"__nvvm_ui2f_rz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2f_rm : GCCBuiltin<"__nvvm_ui2f_rm">, + def int_nvvm_ui2f_rm : ClangBuiltin<"__nvvm_ui2f_rm">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ui2f_rp : GCCBuiltin<"__nvvm_ui2f_rp">, + def int_nvvm_ui2f_rp : ClangBuiltin<"__nvvm_ui2f_rp">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_lohi_i2d : GCCBuiltin<"__nvvm_lohi_i2d">, + def int_nvvm_lohi_i2d : ClangBuiltin<"__nvvm_lohi_i2d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, Commutative]>; - def int_nvvm_d2i_lo : GCCBuiltin<"__nvvm_d2i_lo">, + def int_nvvm_d2i_lo : ClangBuiltin<"__nvvm_d2i_lo">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2i_hi : GCCBuiltin<"__nvvm_d2i_hi">, + def int_nvvm_d2i_hi : ClangBuiltin<"__nvvm_d2i_hi">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rn_ftz : GCCBuiltin<"__nvvm_f2ll_rn_ftz">, + def int_nvvm_f2ll_rn_ftz : ClangBuiltin<"__nvvm_f2ll_rn_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rn : GCCBuiltin<"__nvvm_f2ll_rn">, + def int_nvvm_f2ll_rn : ClangBuiltin<"__nvvm_f2ll_rn">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rz_ftz : GCCBuiltin<"__nvvm_f2ll_rz_ftz">, + def int_nvvm_f2ll_rz_ftz : ClangBuiltin<"__nvvm_f2ll_rz_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rz : GCCBuiltin<"__nvvm_f2ll_rz">, + def int_nvvm_f2ll_rz : ClangBuiltin<"__nvvm_f2ll_rz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rm_ftz : GCCBuiltin<"__nvvm_f2ll_rm_ftz">, + def int_nvvm_f2ll_rm_ftz : ClangBuiltin<"__nvvm_f2ll_rm_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rm : GCCBuiltin<"__nvvm_f2ll_rm">, + def int_nvvm_f2ll_rm : ClangBuiltin<"__nvvm_f2ll_rm">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rp_ftz : GCCBuiltin<"__nvvm_f2ll_rp_ftz">, + def int_nvvm_f2ll_rp_ftz : ClangBuiltin<"__nvvm_f2ll_rp_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ll_rp : GCCBuiltin<"__nvvm_f2ll_rp">, + def int_nvvm_f2ll_rp : ClangBuiltin<"__nvvm_f2ll_rp">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rn_ftz : GCCBuiltin<"__nvvm_f2ull_rn_ftz">, + def int_nvvm_f2ull_rn_ftz : ClangBuiltin<"__nvvm_f2ull_rn_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rn : GCCBuiltin<"__nvvm_f2ull_rn">, + def int_nvvm_f2ull_rn : ClangBuiltin<"__nvvm_f2ull_rn">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rz_ftz : GCCBuiltin<"__nvvm_f2ull_rz_ftz">, + def int_nvvm_f2ull_rz_ftz : ClangBuiltin<"__nvvm_f2ull_rz_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rz : GCCBuiltin<"__nvvm_f2ull_rz">, + def int_nvvm_f2ull_rz : ClangBuiltin<"__nvvm_f2ull_rz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rm_ftz : GCCBuiltin<"__nvvm_f2ull_rm_ftz">, + def int_nvvm_f2ull_rm_ftz : ClangBuiltin<"__nvvm_f2ull_rm_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rm : GCCBuiltin<"__nvvm_f2ull_rm">, + def int_nvvm_f2ull_rm : ClangBuiltin<"__nvvm_f2ull_rm">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rp_ftz : GCCBuiltin<"__nvvm_f2ull_rp_ftz">, + def int_nvvm_f2ull_rp_ftz : ClangBuiltin<"__nvvm_f2ull_rp_ftz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2ull_rp : GCCBuiltin<"__nvvm_f2ull_rp">, + def int_nvvm_f2ull_rp : ClangBuiltin<"__nvvm_f2ull_rp">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ll_rn : GCCBuiltin<"__nvvm_d2ll_rn">, + def int_nvvm_d2ll_rn : ClangBuiltin<"__nvvm_d2ll_rn">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ll_rz : GCCBuiltin<"__nvvm_d2ll_rz">, + def int_nvvm_d2ll_rz : ClangBuiltin<"__nvvm_d2ll_rz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ll_rm : GCCBuiltin<"__nvvm_d2ll_rm">, + def int_nvvm_d2ll_rm : ClangBuiltin<"__nvvm_d2ll_rm">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ll_rp : GCCBuiltin<"__nvvm_d2ll_rp">, + def int_nvvm_d2ll_rp : ClangBuiltin<"__nvvm_d2ll_rp">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ull_rn : GCCBuiltin<"__nvvm_d2ull_rn">, + def int_nvvm_d2ull_rn : ClangBuiltin<"__nvvm_d2ull_rn">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ull_rz : GCCBuiltin<"__nvvm_d2ull_rz">, + def int_nvvm_d2ull_rz : ClangBuiltin<"__nvvm_d2ull_rz">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ull_rm : GCCBuiltin<"__nvvm_d2ull_rm">, + def int_nvvm_d2ull_rm : ClangBuiltin<"__nvvm_d2ull_rm">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_d2ull_rp : GCCBuiltin<"__nvvm_d2ull_rp">, + def int_nvvm_d2ull_rp : ClangBuiltin<"__nvvm_d2ull_rp">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2f_rn : GCCBuiltin<"__nvvm_ll2f_rn">, + def int_nvvm_ll2f_rn : ClangBuiltin<"__nvvm_ll2f_rn">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2f_rz : GCCBuiltin<"__nvvm_ll2f_rz">, + def int_nvvm_ll2f_rz : ClangBuiltin<"__nvvm_ll2f_rz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2f_rm : GCCBuiltin<"__nvvm_ll2f_rm">, + def int_nvvm_ll2f_rm : ClangBuiltin<"__nvvm_ll2f_rm">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2f_rp : GCCBuiltin<"__nvvm_ll2f_rp">, + def int_nvvm_ll2f_rp : ClangBuiltin<"__nvvm_ll2f_rp">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2f_rn : GCCBuiltin<"__nvvm_ull2f_rn">, + def int_nvvm_ull2f_rn : ClangBuiltin<"__nvvm_ull2f_rn">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2f_rz : GCCBuiltin<"__nvvm_ull2f_rz">, + def int_nvvm_ull2f_rz : ClangBuiltin<"__nvvm_ull2f_rz">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2f_rm : GCCBuiltin<"__nvvm_ull2f_rm">, + def int_nvvm_ull2f_rm : ClangBuiltin<"__nvvm_ull2f_rm">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2f_rp : GCCBuiltin<"__nvvm_ull2f_rp">, + def int_nvvm_ull2f_rp : ClangBuiltin<"__nvvm_ull2f_rp">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2d_rn : GCCBuiltin<"__nvvm_ll2d_rn">, + def int_nvvm_ll2d_rn : ClangBuiltin<"__nvvm_ll2d_rn">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2d_rz : GCCBuiltin<"__nvvm_ll2d_rz">, + def int_nvvm_ll2d_rz : ClangBuiltin<"__nvvm_ll2d_rz">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2d_rm : GCCBuiltin<"__nvvm_ll2d_rm">, + def int_nvvm_ll2d_rm : ClangBuiltin<"__nvvm_ll2d_rm">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ll2d_rp : GCCBuiltin<"__nvvm_ll2d_rp">, + def int_nvvm_ll2d_rp : ClangBuiltin<"__nvvm_ll2d_rp">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2d_rn : GCCBuiltin<"__nvvm_ull2d_rn">, + def int_nvvm_ull2d_rn : ClangBuiltin<"__nvvm_ull2d_rn">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2d_rz : GCCBuiltin<"__nvvm_ull2d_rz">, + def int_nvvm_ull2d_rz : ClangBuiltin<"__nvvm_ull2d_rz">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2d_rm : GCCBuiltin<"__nvvm_ull2d_rm">, + def int_nvvm_ull2d_rm : ClangBuiltin<"__nvvm_ull2d_rm">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ull2d_rp : GCCBuiltin<"__nvvm_ull2d_rp">, + def int_nvvm_ull2d_rp : ClangBuiltin<"__nvvm_ull2d_rp">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2h_rn_ftz : GCCBuiltin<"__nvvm_f2h_rn_ftz">, + def int_nvvm_f2h_rn_ftz : ClangBuiltin<"__nvvm_f2h_rn_ftz">, DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_f2h_rn : GCCBuiltin<"__nvvm_f2h_rn">, + def int_nvvm_f2h_rn : ClangBuiltin<"__nvvm_f2h_rn">, DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_ff2bf16x2_rn : GCCBuiltin<"__nvvm_ff2bf16x2_rn">, - Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2bf16x2_rn_relu : GCCBuiltin<"__nvvm_ff2bf16x2_rn_relu">, - Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2bf16x2_rz : GCCBuiltin<"__nvvm_ff2bf16x2_rz">, - Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2bf16x2_rz_relu : GCCBuiltin<"__nvvm_ff2bf16x2_rz_relu">, + def int_nvvm_ff2bf16x2_rn : ClangBuiltin<"__nvvm_ff2bf16x2_rn">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2bf16x2_rn_relu : ClangBuiltin<"__nvvm_ff2bf16x2_rn_relu">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2bf16x2_rz : ClangBuiltin<"__nvvm_ff2bf16x2_rz">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2bf16x2_rz_relu : ClangBuiltin<"__nvvm_ff2bf16x2_rz_relu">, Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2f16x2_rn : GCCBuiltin<"__nvvm_ff2f16x2_rn">, - Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2f16x2_rn_relu : GCCBuiltin<"__nvvm_ff2f16x2_rn_relu">, - Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2f16x2_rz : GCCBuiltin<"__nvvm_ff2f16x2_rz">, - Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_nvvm_ff2f16x2_rz_relu : GCCBuiltin<"__nvvm_ff2f16x2_rz_relu">, - Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - - def int_nvvm_f2bf16_rn : GCCBuiltin<"__nvvm_f2bf16_rn">, - Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_f2bf16_rn_relu : GCCBuiltin<"__nvvm_f2bf16_rn_relu">, - Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_f2bf16_rz : GCCBuiltin<"__nvvm_f2bf16_rz">, - Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; - def int_nvvm_f2bf16_rz_relu : GCCBuiltin<"__nvvm_f2bf16_rz_relu">, - Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; - - def int_nvvm_f2tf32_rna : GCCBuiltin<"__nvvm_f2tf32_rna">, - Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2f16x2_rn : ClangBuiltin<"__nvvm_ff2f16x2_rn">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2f16x2_rn_relu : ClangBuiltin<"__nvvm_ff2f16x2_rn_relu">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2f16x2_rz : ClangBuiltin<"__nvvm_ff2f16x2_rz">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_ff2f16x2_rz_relu : ClangBuiltin<"__nvvm_ff2f16x2_rz_relu">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + + def int_nvvm_f2bf16_rn : ClangBuiltin<"__nvvm_f2bf16_rn">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_f2bf16_rn_relu : ClangBuiltin<"__nvvm_f2bf16_rn_relu">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_f2bf16_rz : ClangBuiltin<"__nvvm_f2bf16_rz">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + def int_nvvm_f2bf16_rz_relu : ClangBuiltin<"__nvvm_f2bf16_rz_relu">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>; + + def int_nvvm_f2tf32_rna : ClangBuiltin<"__nvvm_f2tf32_rna">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>; // // Bitcast // - def int_nvvm_bitcast_f2i : GCCBuiltin<"__nvvm_bitcast_f2i">, + def int_nvvm_bitcast_f2i : ClangBuiltin<"__nvvm_bitcast_f2i">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_bitcast_i2f : GCCBuiltin<"__nvvm_bitcast_i2f">, + def int_nvvm_bitcast_i2f : ClangBuiltin<"__nvvm_bitcast_i2f">, DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_bitcast_ll2d : GCCBuiltin<"__nvvm_bitcast_ll2d">, + def int_nvvm_bitcast_ll2d : ClangBuiltin<"__nvvm_bitcast_ll2d">, DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable]>; - def int_nvvm_bitcast_d2ll : GCCBuiltin<"__nvvm_bitcast_d2ll">, + def int_nvvm_bitcast_d2ll : ClangBuiltin<"__nvvm_bitcast_d2ll">, DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>; // FNS - def int_nvvm_fns : GCCBuiltin<"__nvvm_fns">, + def int_nvvm_fns : ClangBuiltin<"__nvvm_fns">, DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; // Atomics not available as llvm intrinsics. def int_nvvm_atomic_load_inc_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType<llvm_i32_ty>, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; + [IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>]>; def int_nvvm_atomic_load_dec_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType<llvm_i32_ty>, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; + [IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>]>; class SCOPED_ATOMIC2_impl<LLVMType elty> : Intrinsic<[elty], [LLVMAnyPointerType<LLVMMatchType<0>>, LLVMMatchType<0>], - [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; + [IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>]>; class SCOPED_ATOMIC3_impl<LLVMType elty> : Intrinsic<[elty], [LLVMAnyPointerType<LLVMMatchType<0>>, LLVMMatchType<0>, LLVMMatchType<0>], - [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; + [IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>]>; multiclass PTXAtomicWithScope2<LLVMType elty> { def _cta : SCOPED_ATOMIC2_impl<elty>; @@ -1280,177 +1331,179 @@ let TargetPrefix = "nvvm" in { // The builtin for "bar.sync 0" is called __syncthreads. Unlike most of the // intrinsics in this file, this one is a user-facing API. - def int_nvvm_barrier0 : GCCBuiltin<"__syncthreads">, - Intrinsic<[], [], [IntrConvergent]>; + def int_nvvm_barrier0 : ClangBuiltin<"__syncthreads">, + Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>; // Synchronize all threads in the CTA at barrier 'n'. - def int_nvvm_barrier_n : GCCBuiltin<"__nvvm_bar_n">, - Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>; + def int_nvvm_barrier_n : ClangBuiltin<"__nvvm_bar_n">, + Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>; // Synchronize 'm', a multiple of warp size, (arg 2) threads in // the CTA at barrier 'n' (arg 1). - def int_nvvm_barrier : GCCBuiltin<"__nvvm_bar">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>; - def int_nvvm_barrier0_popc : GCCBuiltin<"__nvvm_bar0_popc">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; - def int_nvvm_barrier0_and : GCCBuiltin<"__nvvm_bar0_and">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; - def int_nvvm_barrier0_or : GCCBuiltin<"__nvvm_bar0_or">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; + def int_nvvm_barrier : ClangBuiltin<"__nvvm_bar">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrNoCallback]>; + def int_nvvm_barrier0_popc : ClangBuiltin<"__nvvm_bar0_popc">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>; + def int_nvvm_barrier0_and : ClangBuiltin<"__nvvm_bar0_and">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>; + def int_nvvm_barrier0_or : ClangBuiltin<"__nvvm_bar0_or">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>; def int_nvvm_bar_sync : - Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, - GCCBuiltin<"__nvvm_bar_sync">; + Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>, + ClangBuiltin<"__nvvm_bar_sync">; def int_nvvm_bar_warp_sync : - Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, - GCCBuiltin<"__nvvm_bar_warp_sync">; + Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>, + ClangBuiltin<"__nvvm_bar_warp_sync">; // barrier.sync id[, cnt] def int_nvvm_barrier_sync : - Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, - GCCBuiltin<"__nvvm_barrier_sync">; + Intrinsic<[], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>, + ClangBuiltin<"__nvvm_barrier_sync">; def int_nvvm_barrier_sync_cnt : - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>, - GCCBuiltin<"__nvvm_barrier_sync_cnt">; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrNoCallback]>, + ClangBuiltin<"__nvvm_barrier_sync_cnt">; // Membar - def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">, - Intrinsic<[], [], []>; - def int_nvvm_membar_gl : GCCBuiltin<"__nvvm_membar_gl">, - Intrinsic<[], [], []>; - def int_nvvm_membar_sys : GCCBuiltin<"__nvvm_membar_sys">, - Intrinsic<[], [], []>; + def int_nvvm_membar_cta : ClangBuiltin<"__nvvm_membar_cta">, + Intrinsic<[], [], [IntrNoCallback]>; + def int_nvvm_membar_gl : ClangBuiltin<"__nvvm_membar_gl">, + Intrinsic<[], [], [IntrNoCallback]>; + def int_nvvm_membar_sys : ClangBuiltin<"__nvvm_membar_sys">, + Intrinsic<[], [], [IntrNoCallback]>; // Async Copy def int_nvvm_cp_async_mbarrier_arrive : - GCCBuiltin<"__nvvm_cp_async_mbarrier_arrive">, - Intrinsic<[],[llvm_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_cp_async_mbarrier_arrive">, + Intrinsic<[],[llvm_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_cp_async_mbarrier_arrive_shared : - GCCBuiltin<"__nvvm_cp_async_mbarrier_arrive_shared">, - Intrinsic<[],[llvm_shared_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_cp_async_mbarrier_arrive_shared">, + Intrinsic<[],[llvm_shared_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_cp_async_mbarrier_arrive_noinc : - GCCBuiltin<"__nvvm_cp_async_mbarrier_arrive_noinc">, - Intrinsic<[],[llvm_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_cp_async_mbarrier_arrive_noinc">, + Intrinsic<[],[llvm_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_cp_async_mbarrier_arrive_noinc_shared : - GCCBuiltin<"__nvvm_cp_async_mbarrier_arrive_noinc_shared">, - Intrinsic<[],[llvm_shared_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_cp_async_mbarrier_arrive_noinc_shared">, + Intrinsic<[],[llvm_shared_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_cp_async_ca_shared_global_4 : - GCCBuiltin<"__nvvm_cp_async_ca_shared_global_4">, + ClangBuiltin<"__nvvm_cp_async_ca_shared_global_4">, Intrinsic<[],[llvm_shared_i8ptr_ty, llvm_global_i8ptr_ty], - [IntrArgMemOnly, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, + [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>], "llvm.nvvm.cp.async.ca.shared.global.4">; def int_nvvm_cp_async_ca_shared_global_8 : - GCCBuiltin<"__nvvm_cp_async_ca_shared_global_8">, + ClangBuiltin<"__nvvm_cp_async_ca_shared_global_8">, Intrinsic<[],[llvm_shared_i8ptr_ty, llvm_global_i8ptr_ty], - [IntrArgMemOnly, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, + [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>], "llvm.nvvm.cp.async.ca.shared.global.8">; def int_nvvm_cp_async_ca_shared_global_16 : - GCCBuiltin<"__nvvm_cp_async_ca_shared_global_16">, + ClangBuiltin<"__nvvm_cp_async_ca_shared_global_16">, Intrinsic<[],[llvm_shared_i8ptr_ty, llvm_global_i8ptr_ty], - [IntrArgMemOnly, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, + [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>], "llvm.nvvm.cp.async.ca.shared.global.16">; def int_nvvm_cp_async_cg_shared_global_16 : - GCCBuiltin<"__nvvm_cp_async_cg_shared_global_16">, + ClangBuiltin<"__nvvm_cp_async_cg_shared_global_16">, Intrinsic<[],[llvm_shared_i8ptr_ty, llvm_global_i8ptr_ty], - [IntrArgMemOnly, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, + [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>], "llvm.nvvm.cp.async.cg.shared.global.16">; def int_nvvm_cp_async_commit_group : - GCCBuiltin<"__nvvm_cp_async_commit_group">, + ClangBuiltin<"__nvvm_cp_async_commit_group">, Intrinsic<[],[],[]>; def int_nvvm_cp_async_wait_group : - GCCBuiltin<"__nvvm_cp_async_wait_group">, + ClangBuiltin<"__nvvm_cp_async_wait_group">, Intrinsic<[],[llvm_i32_ty],[ImmArg<ArgIndex<0>>]>; def int_nvvm_cp_async_wait_all : - GCCBuiltin<"__nvvm_cp_async_wait_all">, + ClangBuiltin<"__nvvm_cp_async_wait_all">, Intrinsic<[],[],[]>; // mbarrier -def int_nvvm_mbarrier_init : GCCBuiltin<"__nvvm_mbarrier_init">, - Intrinsic<[],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; +def int_nvvm_mbarrier_init : ClangBuiltin<"__nvvm_mbarrier_init">, + Intrinsic<[],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_init_shared : - GCCBuiltin<"__nvvm_mbarrier_init_shared">, - Intrinsic<[],[llvm_shared_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_init_shared">, + Intrinsic<[],[llvm_shared_i64ptr_ty, llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; -def int_nvvm_mbarrier_inval : GCCBuiltin<"__nvvm_mbarrier_inval">, +def int_nvvm_mbarrier_inval : ClangBuiltin<"__nvvm_mbarrier_inval">, Intrinsic<[],[llvm_i64ptr_ty], - [IntrConvergent, IntrWriteMem, IntrArgMemOnly, + [IntrConvergent, IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>; def int_nvvm_mbarrier_inval_shared : - GCCBuiltin<"__nvvm_mbarrier_inval_shared">, + ClangBuiltin<"__nvvm_mbarrier_inval_shared">, Intrinsic<[],[llvm_shared_i64ptr_ty], - [IntrConvergent, IntrWriteMem, IntrArgMemOnly, + [IntrConvergent, IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>; -def int_nvvm_mbarrier_arrive : GCCBuiltin<"__nvvm_mbarrier_arrive">, - Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty],[IntrConvergent]>; +def int_nvvm_mbarrier_arrive : ClangBuiltin<"__nvvm_mbarrier_arrive">, + Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_shared : - GCCBuiltin<"__nvvm_mbarrier_arrive_shared">, - Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_shared">, + Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_noComplete : - GCCBuiltin<"__nvvm_mbarrier_arrive_noComplete">, - Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_noComplete">, + Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_noComplete_shared : - GCCBuiltin<"__nvvm_mbarrier_arrive_noComplete_shared">, - Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_noComplete_shared">, + Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty, + llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_drop : - GCCBuiltin<"__nvvm_mbarrier_arrive_drop">, - Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_drop">, + Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_drop_shared : - GCCBuiltin<"__nvvm_mbarrier_arrive_drop_shared">, - Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_drop_shared">, + Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_drop_noComplete : - GCCBuiltin<"__nvvm_mbarrier_arrive_drop_noComplete">, - Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_drop_noComplete">, + Intrinsic<[llvm_i64_ty],[llvm_i64ptr_ty, llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_arrive_drop_noComplete_shared : - GCCBuiltin<"__nvvm_mbarrier_arrive_drop_noComplete_shared">, - Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty, llvm_i32_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_arrive_drop_noComplete_shared">, + Intrinsic<[llvm_i64_ty],[llvm_shared_i64ptr_ty, + llvm_i32_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_test_wait : - GCCBuiltin<"__nvvm_mbarrier_test_wait">, - Intrinsic<[llvm_i1_ty],[llvm_i64ptr_ty, llvm_i64_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_test_wait">, + Intrinsic<[llvm_i1_ty],[llvm_i64ptr_ty, llvm_i64_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_test_wait_shared : - GCCBuiltin<"__nvvm_mbarrier_test_wait_shared">, - Intrinsic<[llvm_i1_ty],[llvm_shared_i64ptr_ty, llvm_i64_ty],[IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_test_wait_shared">, + Intrinsic<[llvm_i1_ty],[llvm_shared_i64ptr_ty, llvm_i64_ty],[IntrConvergent, IntrNoCallback]>; def int_nvvm_mbarrier_pending_count : - GCCBuiltin<"__nvvm_mbarrier_pending_count">, - Intrinsic<[llvm_i32_ty],[llvm_i64_ty],[IntrNoMem, IntrConvergent]>; + ClangBuiltin<"__nvvm_mbarrier_pending_count">, + Intrinsic<[llvm_i32_ty],[llvm_i64_ty],[IntrNoMem, IntrConvergent, IntrNoCallback]>; // Generated within nvvm. Use for ldu on sm_20 or later. Second arg is the // pointer's alignment. def int_nvvm_ldu_global_i : Intrinsic<[llvm_anyint_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldu.global.i">; def int_nvvm_ldu_global_f : Intrinsic<[llvm_anyfloat_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldu.global.f">; def int_nvvm_ldu_global_p : Intrinsic<[llvm_anyptr_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldu.global.p">; // Generated within nvvm. Use for ldg on sm_35 or later. Second arg is the // pointer's alignment. def int_nvvm_ldg_global_i : Intrinsic<[llvm_anyint_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldg.global.i">; def int_nvvm_ldg_global_f : Intrinsic<[llvm_anyfloat_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldg.global.f">; def int_nvvm_ldg_global_p : Intrinsic<[llvm_anyptr_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, NoCapture<ArgIndex<0>>], "llvm.nvvm.ldg.global.p">; // Use for generic pointers @@ -1491,7 +1544,7 @@ def int_nvvm_ptr_gen_to_constant: DefaultAttrsIntrinsic<[llvm_anyptr_ty], // This is for params that are passed to kernel functions by pointer by-val. def int_nvvm_ptr_gen_to_param: Intrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty], - [IntrNoMem, IntrSpeculatable], + [IntrNoMem, IntrSpeculatable, IntrNoCallback], "llvm.nvvm.ptr.gen.to.param">; // Move intrinsics, used in nvvm internally @@ -1531,149 +1584,149 @@ def int_nvvm_reflect : def int_nvvm_isspacep_const : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.isspacep.const">, - GCCBuiltin<"__nvvm_isspacep_const">; + ClangBuiltin<"__nvvm_isspacep_const">; def int_nvvm_isspacep_global : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.isspacep.global">, - GCCBuiltin<"__nvvm_isspacep_global">; + ClangBuiltin<"__nvvm_isspacep_global">; def int_nvvm_isspacep_local : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.isspacep.local">, - GCCBuiltin<"__nvvm_isspacep_local">; + ClangBuiltin<"__nvvm_isspacep_local">; def int_nvvm_isspacep_shared : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.isspacep.shared">, - GCCBuiltin<"__nvvm_isspacep_shared">; + ClangBuiltin<"__nvvm_isspacep_shared">; // Environment register read def int_nvvm_read_ptx_sreg_envreg0 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg0">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg0">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg0">; def int_nvvm_read_ptx_sreg_envreg1 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg1">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg1">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg1">; def int_nvvm_read_ptx_sreg_envreg2 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg2">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg2">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg2">; def int_nvvm_read_ptx_sreg_envreg3 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg3">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg3">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg3">; def int_nvvm_read_ptx_sreg_envreg4 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg4">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg4">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg4">; def int_nvvm_read_ptx_sreg_envreg5 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg5">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg5">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg5">; def int_nvvm_read_ptx_sreg_envreg6 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg6">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg6">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg6">; def int_nvvm_read_ptx_sreg_envreg7 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg7">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg7">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg7">; def int_nvvm_read_ptx_sreg_envreg8 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg8">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg8">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg8">; def int_nvvm_read_ptx_sreg_envreg9 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg9">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg9">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg9">; def int_nvvm_read_ptx_sreg_envreg10 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg10">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg10">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg10">; def int_nvvm_read_ptx_sreg_envreg11 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg11">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg11">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg11">; def int_nvvm_read_ptx_sreg_envreg12 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg12">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg12">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg12">; def int_nvvm_read_ptx_sreg_envreg13 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg13">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg13">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg13">; def int_nvvm_read_ptx_sreg_envreg14 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg14">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg14">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg14">; def int_nvvm_read_ptx_sreg_envreg15 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg15">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg15">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg15">; def int_nvvm_read_ptx_sreg_envreg16 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg16">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg16">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg16">; def int_nvvm_read_ptx_sreg_envreg17 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg17">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg17">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg17">; def int_nvvm_read_ptx_sreg_envreg18 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg18">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg18">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg18">; def int_nvvm_read_ptx_sreg_envreg19 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg19">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg19">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg19">; def int_nvvm_read_ptx_sreg_envreg20 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg20">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg20">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg20">; def int_nvvm_read_ptx_sreg_envreg21 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg21">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg21">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg21">; def int_nvvm_read_ptx_sreg_envreg22 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg22">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg22">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg22">; def int_nvvm_read_ptx_sreg_envreg23 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg23">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg23">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg23">; def int_nvvm_read_ptx_sreg_envreg24 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg24">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg24">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg24">; def int_nvvm_read_ptx_sreg_envreg25 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg25">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg25">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg25">; def int_nvvm_read_ptx_sreg_envreg26 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg26">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg26">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg26">; def int_nvvm_read_ptx_sreg_envreg27 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg27">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg27">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg27">; def int_nvvm_read_ptx_sreg_envreg28 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg28">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg28">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg28">; def int_nvvm_read_ptx_sreg_envreg29 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg29">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg29">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg29">; def int_nvvm_read_ptx_sreg_envreg30 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg30">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg30">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg30">; def int_nvvm_read_ptx_sreg_envreg31 : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.read.ptx.sreg.envreg31">, - GCCBuiltin<"__nvvm_read_ptx_sreg_envreg31">; + ClangBuiltin<"__nvvm_read_ptx_sreg_envreg31">; // Texture Fetch @@ -3161,62 +3214,62 @@ def int_nvvm_suld_3d_v4i32_zero def int_nvvm_txq_channel_order : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.channel.order">, - GCCBuiltin<"__nvvm_txq_channel_order">; + ClangBuiltin<"__nvvm_txq_channel_order">; def int_nvvm_txq_channel_data_type : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.channel.data.type">, - GCCBuiltin<"__nvvm_txq_channel_data_type">; + ClangBuiltin<"__nvvm_txq_channel_data_type">; def int_nvvm_txq_width : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.width">, - GCCBuiltin<"__nvvm_txq_width">; + ClangBuiltin<"__nvvm_txq_width">; def int_nvvm_txq_height : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.height">, - GCCBuiltin<"__nvvm_txq_height">; + ClangBuiltin<"__nvvm_txq_height">; def int_nvvm_txq_depth : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.depth">, - GCCBuiltin<"__nvvm_txq_depth">; + ClangBuiltin<"__nvvm_txq_depth">; def int_nvvm_txq_array_size : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.array.size">, - GCCBuiltin<"__nvvm_txq_array_size">; + ClangBuiltin<"__nvvm_txq_array_size">; def int_nvvm_txq_num_samples : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.num.samples">, - GCCBuiltin<"__nvvm_txq_num_samples">; + ClangBuiltin<"__nvvm_txq_num_samples">; def int_nvvm_txq_num_mipmap_levels : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.txq.num.mipmap.levels">, - GCCBuiltin<"__nvvm_txq_num_mipmap_levels">; + ClangBuiltin<"__nvvm_txq_num_mipmap_levels">; //===- Surface Query ------------------------------------------------------===// def int_nvvm_suq_channel_order : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.channel.order">, - GCCBuiltin<"__nvvm_suq_channel_order">; + ClangBuiltin<"__nvvm_suq_channel_order">; def int_nvvm_suq_channel_data_type : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.channel.data.type">, - GCCBuiltin<"__nvvm_suq_channel_data_type">; + ClangBuiltin<"__nvvm_suq_channel_data_type">; def int_nvvm_suq_width : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.width">, - GCCBuiltin<"__nvvm_suq_width">; + ClangBuiltin<"__nvvm_suq_width">; def int_nvvm_suq_height : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.height">, - GCCBuiltin<"__nvvm_suq_height">; + ClangBuiltin<"__nvvm_suq_height">; def int_nvvm_suq_depth : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.depth">, - GCCBuiltin<"__nvvm_suq_depth">; + ClangBuiltin<"__nvvm_suq_depth">; def int_nvvm_suq_array_size : Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.suq.array.size">, - GCCBuiltin<"__nvvm_suq_array_size">; + ClangBuiltin<"__nvvm_suq_array_size">; //===- Handle Query -------------------------------------------------------===// @@ -3224,15 +3277,15 @@ def int_nvvm_suq_array_size def int_nvvm_istypep_sampler : Intrinsic<[llvm_i1_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.istypep.sampler">, - GCCBuiltin<"__nvvm_istypep_sampler">; + ClangBuiltin<"__nvvm_istypep_sampler">; def int_nvvm_istypep_surface : Intrinsic<[llvm_i1_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.istypep.surface">, - GCCBuiltin<"__nvvm_istypep_surface">; + ClangBuiltin<"__nvvm_istypep_surface">; def int_nvvm_istypep_texture : Intrinsic<[llvm_i1_ty], [llvm_i64_ty], [IntrNoMem], "llvm.nvvm.istypep.texture">, - GCCBuiltin<"__nvvm_istypep_texture">; + ClangBuiltin<"__nvvm_istypep_texture">; @@ -3243,810 +3296,810 @@ def int_nvvm_istypep_texture def int_nvvm_sust_b_1d_i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_i8_clamp">; def int_nvvm_sust_b_1d_i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_i16_clamp">; def int_nvvm_sust_b_1d_i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_i32_clamp">; def int_nvvm_sust_b_1d_i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_i64_clamp">; def int_nvvm_sust_b_1d_v2i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i8_clamp">; def int_nvvm_sust_b_1d_v2i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i16_clamp">; def int_nvvm_sust_b_1d_v2i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v2i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i32_clamp">; def int_nvvm_sust_b_1d_v2i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.v2i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i64_clamp">; def int_nvvm_sust_b_1d_v4i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i8_clamp">; def int_nvvm_sust_b_1d_v4i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i16_clamp">; def int_nvvm_sust_b_1d_v4i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v4i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i32_clamp">; def int_nvvm_sust_b_1d_array_i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i8_clamp">; def int_nvvm_sust_b_1d_array_i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i16_clamp">; def int_nvvm_sust_b_1d_array_i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i32_clamp">; def int_nvvm_sust_b_1d_array_i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i64_clamp">; def int_nvvm_sust_b_1d_array_v2i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i8_clamp">; def int_nvvm_sust_b_1d_array_v2i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i16_clamp">; def int_nvvm_sust_b_1d_array_v2i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v2i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i32_clamp">; def int_nvvm_sust_b_1d_array_v2i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.v2i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i64_clamp">; def int_nvvm_sust_b_1d_array_v4i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i8_clamp">; def int_nvvm_sust_b_1d_array_v4i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i16_clamp">; def int_nvvm_sust_b_1d_array_v4i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v4i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i32_clamp">; def int_nvvm_sust_b_2d_i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_i8_clamp">; def int_nvvm_sust_b_2d_i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_i16_clamp">; def int_nvvm_sust_b_2d_i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_i32_clamp">; def int_nvvm_sust_b_2d_i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_i64_clamp">; def int_nvvm_sust_b_2d_v2i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i8_clamp">; def int_nvvm_sust_b_2d_v2i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i16_clamp">; def int_nvvm_sust_b_2d_v2i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v2i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i32_clamp">; def int_nvvm_sust_b_2d_v2i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.v2i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i64_clamp">; def int_nvvm_sust_b_2d_v4i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i8_clamp">; def int_nvvm_sust_b_2d_v4i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i16_clamp">; def int_nvvm_sust_b_2d_v4i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v4i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i32_clamp">; def int_nvvm_sust_b_2d_array_i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i8_clamp">; def int_nvvm_sust_b_2d_array_i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i16_clamp">; def int_nvvm_sust_b_2d_array_i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i32_clamp">; def int_nvvm_sust_b_2d_array_i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i64_clamp">; def int_nvvm_sust_b_2d_array_v2i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i8_clamp">; def int_nvvm_sust_b_2d_array_v2i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i16_clamp">; def int_nvvm_sust_b_2d_array_v2i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v2i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i32_clamp">; def int_nvvm_sust_b_2d_array_v2i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.v2i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i64_clamp">; def int_nvvm_sust_b_2d_array_v4i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i8_clamp">; def int_nvvm_sust_b_2d_array_v4i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i16_clamp">; def int_nvvm_sust_b_2d_array_v4i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v4i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i32_clamp">; def int_nvvm_sust_b_3d_i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_i8_clamp">; def int_nvvm_sust_b_3d_i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_i16_clamp">; def int_nvvm_sust_b_3d_i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_i32_clamp">; def int_nvvm_sust_b_3d_i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_i64_clamp">; def int_nvvm_sust_b_3d_v2i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i8_clamp">; def int_nvvm_sust_b_3d_v2i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i16_clamp">; def int_nvvm_sust_b_3d_v2i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v2i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i32_clamp">; def int_nvvm_sust_b_3d_v2i64_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.v2i64.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i64_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i64_clamp">; def int_nvvm_sust_b_3d_v4i8_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i8.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i8_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i8_clamp">; def int_nvvm_sust_b_3d_v4i16_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i16.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i16_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i16_clamp">; def int_nvvm_sust_b_3d_v4i32_clamp : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v4i32.clamp">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i32_clamp">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i32_clamp">; // .trap variant def int_nvvm_sust_b_1d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_i8_trap">; def int_nvvm_sust_b_1d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_i16_trap">; def int_nvvm_sust_b_1d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_i32_trap">; def int_nvvm_sust_b_1d_i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.i64.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_i64_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_i64_trap">; def int_nvvm_sust_b_1d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i8_trap">; def int_nvvm_sust_b_1d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i16_trap">; def int_nvvm_sust_b_1d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i32_trap">; def int_nvvm_sust_b_1d_v2i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.v2i64.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i64_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i64_trap">; def int_nvvm_sust_b_1d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i8_trap">; def int_nvvm_sust_b_1d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i16_trap">; def int_nvvm_sust_b_1d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i32_trap">; def int_nvvm_sust_b_1d_array_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i8_trap">; def int_nvvm_sust_b_1d_array_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i16_trap">; def int_nvvm_sust_b_1d_array_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i32_trap">; def int_nvvm_sust_b_1d_array_i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.i64.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i64_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i64_trap">; def int_nvvm_sust_b_1d_array_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i8_trap">; def int_nvvm_sust_b_1d_array_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i16_trap">; def int_nvvm_sust_b_1d_array_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i32_trap">; def int_nvvm_sust_b_1d_array_v2i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.v2i64.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i64_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i64_trap">; def int_nvvm_sust_b_1d_array_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i8_trap">; def int_nvvm_sust_b_1d_array_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i16_trap">; def int_nvvm_sust_b_1d_array_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i32_trap">; def int_nvvm_sust_b_2d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_i8_trap">; def int_nvvm_sust_b_2d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_i16_trap">; def int_nvvm_sust_b_2d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_i32_trap">; def int_nvvm_sust_b_2d_i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.i64.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_i64_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_i64_trap">; def int_nvvm_sust_b_2d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i8_trap">; def int_nvvm_sust_b_2d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i16_trap">; def int_nvvm_sust_b_2d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i32_trap">; def int_nvvm_sust_b_2d_v2i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.v2i64.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i64_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i64_trap">; def int_nvvm_sust_b_2d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i8_trap">; def int_nvvm_sust_b_2d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i16_trap">; def int_nvvm_sust_b_2d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i32_trap">; def int_nvvm_sust_b_2d_array_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i8_trap">; def int_nvvm_sust_b_2d_array_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i16_trap">; def int_nvvm_sust_b_2d_array_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i32_trap">; def int_nvvm_sust_b_2d_array_i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.i64.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i64_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i64_trap">; def int_nvvm_sust_b_2d_array_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i8_trap">; def int_nvvm_sust_b_2d_array_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i16_trap">; def int_nvvm_sust_b_2d_array_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i32_trap">; def int_nvvm_sust_b_2d_array_v2i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.v2i64.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i64_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i64_trap">; def int_nvvm_sust_b_2d_array_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i8_trap">; def int_nvvm_sust_b_2d_array_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i16_trap">; def int_nvvm_sust_b_2d_array_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i32_trap">; def int_nvvm_sust_b_3d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i8.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_i8_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_i8_trap">; def int_nvvm_sust_b_3d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i16.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_i16_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_i16_trap">; def int_nvvm_sust_b_3d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.i32.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_i32_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_i32_trap">; def int_nvvm_sust_b_3d_i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.i64.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_i64_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_i64_trap">; def int_nvvm_sust_b_3d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i8_trap">; def int_nvvm_sust_b_3d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i16_trap">; def int_nvvm_sust_b_3d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i32_trap">; def int_nvvm_sust_b_3d_v2i64_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.v2i64.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i64_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i64_trap">; def int_nvvm_sust_b_3d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i8_trap">; def int_nvvm_sust_b_3d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i16_trap">; def int_nvvm_sust_b_3d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i32_trap">; // .zero variant def int_nvvm_sust_b_1d_i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_i8_zero">; def int_nvvm_sust_b_1d_i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_i16_zero">; def int_nvvm_sust_b_1d_i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_i32_zero">; def int_nvvm_sust_b_1d_i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.i64.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_i64_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_i64_zero">; def int_nvvm_sust_b_1d_v2i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i8_zero">; def int_nvvm_sust_b_1d_v2i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v2i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i16_zero">; def int_nvvm_sust_b_1d_v2i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v2i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i32_zero">; def int_nvvm_sust_b_1d_v2i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.v2i64.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v2i64_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v2i64_zero">; def int_nvvm_sust_b_1d_v4i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i8_zero">; def int_nvvm_sust_b_1d_v4i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.v4i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i16_zero">; def int_nvvm_sust_b_1d_v4i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.v4i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_v4i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_v4i32_zero">; def int_nvvm_sust_b_1d_array_i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i8_zero">; def int_nvvm_sust_b_1d_array_i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i16_zero">; def int_nvvm_sust_b_1d_array_i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i32_zero">; def int_nvvm_sust_b_1d_array_i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.i64.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_i64_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_i64_zero">; def int_nvvm_sust_b_1d_array_v2i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i8_zero">; def int_nvvm_sust_b_1d_array_v2i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v2i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i16_zero">; def int_nvvm_sust_b_1d_array_v2i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v2i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i32_zero">; def int_nvvm_sust_b_1d_array_v2i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.1d.array.v2i64.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v2i64_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v2i64_zero">; def int_nvvm_sust_b_1d_array_v4i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i8.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i8_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i8_zero">; def int_nvvm_sust_b_1d_array_v4i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.1d.array.v4i16.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i16_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i16_zero">; def int_nvvm_sust_b_1d_array_v4i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.1d.array.v4i32.zero">, - GCCBuiltin<"__nvvm_sust_b_1d_array_v4i32_zero">; + ClangBuiltin<"__nvvm_sust_b_1d_array_v4i32_zero">; def int_nvvm_sust_b_2d_i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_i8_zero">; def int_nvvm_sust_b_2d_i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_i16_zero">; def int_nvvm_sust_b_2d_i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_i32_zero">; def int_nvvm_sust_b_2d_i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.i64.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_i64_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_i64_zero">; def int_nvvm_sust_b_2d_v2i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i8_zero">; def int_nvvm_sust_b_2d_v2i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v2i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i16_zero">; def int_nvvm_sust_b_2d_v2i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v2i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i32_zero">; def int_nvvm_sust_b_2d_v2i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.v2i64.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v2i64_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v2i64_zero">; def int_nvvm_sust_b_2d_v4i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i8_zero">; def int_nvvm_sust_b_2d_v4i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.v4i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i16_zero">; def int_nvvm_sust_b_2d_v4i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.v4i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_v4i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_v4i32_zero">; def int_nvvm_sust_b_2d_array_i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i8_zero">; def int_nvvm_sust_b_2d_array_i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i16_zero">; def int_nvvm_sust_b_2d_array_i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i32_zero">; def int_nvvm_sust_b_2d_array_i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.i64.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_i64_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_i64_zero">; def int_nvvm_sust_b_2d_array_v2i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i8_zero">; def int_nvvm_sust_b_2d_array_v2i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v2i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i16_zero">; def int_nvvm_sust_b_2d_array_v2i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v2i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i32_zero">; def int_nvvm_sust_b_2d_array_v2i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.2d.array.v2i64.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v2i64_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v2i64_zero">; def int_nvvm_sust_b_2d_array_v4i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i8.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i8_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i8_zero">; def int_nvvm_sust_b_2d_array_v4i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.2d.array.v4i16.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i16_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i16_zero">; def int_nvvm_sust_b_2d_array_v4i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.2d.array.v4i32.zero">, - GCCBuiltin<"__nvvm_sust_b_2d_array_v4i32_zero">; + ClangBuiltin<"__nvvm_sust_b_2d_array_v4i32_zero">; def int_nvvm_sust_b_3d_i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i8.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_i8_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_i8_zero">; def int_nvvm_sust_b_3d_i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.i16.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_i16_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_i16_zero">; def int_nvvm_sust_b_3d_i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.i32.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_i32_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_i32_zero">; def int_nvvm_sust_b_3d_i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.i64.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_i64_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_i64_zero">; def int_nvvm_sust_b_3d_v2i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i8.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i8_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i8_zero">; def int_nvvm_sust_b_3d_v2i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v2i16.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i16_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i16_zero">; def int_nvvm_sust_b_3d_v2i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v2i32.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i32_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i32_zero">; def int_nvvm_sust_b_3d_v2i64_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i64_ty, llvm_i64_ty], [], "llvm.nvvm.sust.b.3d.v2i64.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v2i64_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v2i64_zero">; def int_nvvm_sust_b_3d_v4i8_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i8.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i8_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i8_zero">; def int_nvvm_sust_b_3d_v4i16_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.b.3d.v4i16.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i16_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i16_zero">; def int_nvvm_sust_b_3d_v4i32_zero : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.b.3d.v4i32.zero">, - GCCBuiltin<"__nvvm_sust_b_3d_v4i32_zero">; + ClangBuiltin<"__nvvm_sust_b_3d_v4i32_zero">; @@ -4055,245 +4108,245 @@ def int_nvvm_sust_b_3d_v4i32_zero def int_nvvm_sust_p_1d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_i8_trap">; def int_nvvm_sust_p_1d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_i16_trap">; def int_nvvm_sust_p_1d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_i32_trap">; def int_nvvm_sust_p_1d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v2i8_trap">; def int_nvvm_sust_p_1d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v2i16_trap">; def int_nvvm_sust_p_1d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v2i32_trap">; def int_nvvm_sust_p_1d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v4i8_trap">; def int_nvvm_sust_p_1d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v4i16_trap">; def int_nvvm_sust_p_1d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_v4i32_trap">; def int_nvvm_sust_p_1d_array_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_i8_trap">; def int_nvvm_sust_p_1d_array_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_i16_trap">; def int_nvvm_sust_p_1d_array_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.array.i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_i32_trap">; def int_nvvm_sust_p_1d_array_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v2i8_trap">; def int_nvvm_sust_p_1d_array_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v2i16_trap">; def int_nvvm_sust_p_1d_array_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.array.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v2i32_trap">; def int_nvvm_sust_p_1d_array_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v4i8_trap">; def int_nvvm_sust_p_1d_array_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.1d.array.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v4i16_trap">; def int_nvvm_sust_p_1d_array_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.1d.array.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_p_1d_array_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_p_1d_array_v4i32_trap">; def int_nvvm_sust_p_2d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_i8_trap">; def int_nvvm_sust_p_2d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_i16_trap">; def int_nvvm_sust_p_2d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_i32_trap">; def int_nvvm_sust_p_2d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v2i8_trap">; def int_nvvm_sust_p_2d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v2i16_trap">; def int_nvvm_sust_p_2d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v2i32_trap">; def int_nvvm_sust_p_2d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v4i8_trap">; def int_nvvm_sust_p_2d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v4i16_trap">; def int_nvvm_sust_p_2d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_v4i32_trap">; def int_nvvm_sust_p_2d_array_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_i8_trap">; def int_nvvm_sust_p_2d_array_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_i16_trap">; def int_nvvm_sust_p_2d_array_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.array.i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_i32_trap">; def int_nvvm_sust_p_2d_array_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v2i8_trap">; def int_nvvm_sust_p_2d_array_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v2i16_trap">; def int_nvvm_sust_p_2d_array_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.array.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v2i32_trap">; def int_nvvm_sust_p_2d_array_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v4i8_trap">; def int_nvvm_sust_p_2d_array_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.2d.array.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v4i16_trap">; def int_nvvm_sust_p_2d_array_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.2d.array.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_p_2d_array_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_p_2d_array_v4i32_trap">; def int_nvvm_sust_p_3d_i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.i8.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_i8_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_i8_trap">; def int_nvvm_sust_p_3d_i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.i16.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_i16_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_i16_trap">; def int_nvvm_sust_p_3d_i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.3d.i32.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_i32_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_i32_trap">; def int_nvvm_sust_p_3d_v2i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.v2i8.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v2i8_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v2i8_trap">; def int_nvvm_sust_p_3d_v2i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.v2i16.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v2i16_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v2i16_trap">; def int_nvvm_sust_p_3d_v2i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.3d.v2i32.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v2i32_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v2i32_trap">; def int_nvvm_sust_p_3d_v4i8_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.v4i8.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v4i8_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v4i8_trap">; def int_nvvm_sust_p_3d_v4i16_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i16_ty], [], "llvm.nvvm.sust.p.3d.v4i16.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v4i16_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v4i16_trap">; def int_nvvm_sust_p_3d_v4i32_trap : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [], "llvm.nvvm.sust.p.3d.v4i32.trap">, - GCCBuiltin<"__nvvm_sust_p_3d_v4i32_trap">; + ClangBuiltin<"__nvvm_sust_p_3d_v4i32_trap">; def int_nvvm_rotate_b32 : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.b32">, - GCCBuiltin<"__nvvm_rotate_b32">; + ClangBuiltin<"__nvvm_rotate_b32">; def int_nvvm_rotate_b64 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.b64">, - GCCBuiltin<"__nvvm_rotate_b64">; + ClangBuiltin<"__nvvm_rotate_b64">; def int_nvvm_rotate_right_b64 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.right.b64">, - GCCBuiltin<"__nvvm_rotate_right_b64">; + ClangBuiltin<"__nvvm_rotate_right_b64">; def int_nvvm_swap_lo_hi_b64 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem, IntrSpeculatable], "llvm.nvvm.swap.lo.hi.b64">, - GCCBuiltin<"__nvvm_swap_lo_hi_b64">; + ClangBuiltin<"__nvvm_swap_lo_hi_b64">; // Accessing special registers. @@ -4304,31 +4357,31 @@ multiclass PTXReadSRegIntrinsic_v4i32<string regname> { // FIXME: Enable this once v4i32 support is enabled in back-end. // def _v4i16 : Intrinsic<[llvm_v4i32_ty], [], [IntrNoMem, IntrSpeculatable]>; - def _x : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_x">; - def _y : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_y">; - def _z : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_z">; - def _w : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_w">; + def _x : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_x">; + def _y : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_y">; + def _z : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_z">; + def _w : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_w">; } class PTXReadSRegIntrinsic_r32<string name> : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + ClangBuiltin<"__nvvm_read_ptx_sreg_" # name>; class PTXReadSRegIntrinsic_r64<string name> : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrSpeculatable]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + ClangBuiltin<"__nvvm_read_ptx_sreg_" # name>; // Intrinsics to read registers with non-constant values. E.g. the values that // do change over the kernel lifetime. Such reads should not be CSE'd. class PTXReadNCSRegIntrinsic_r32<string name> - : Intrinsic<[llvm_i32_ty], [], [IntrInaccessibleMemOnly]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + : Intrinsic<[llvm_i32_ty], [], [IntrInaccessibleMemOnly, IntrNoCallback]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # name>; class PTXReadNCSRegIntrinsic_r64<string name> - : Intrinsic<[llvm_i64_ty], [], [IntrInaccessibleMemOnly]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + : Intrinsic<[llvm_i64_ty], [], [IntrInaccessibleMemOnly, IntrNoCallback]>, + ClangBuiltin<"__nvvm_read_ptx_sreg_" # name>; defm int_nvvm_read_ptx_sreg_tid : PTXReadSRegIntrinsic_v4i32<"tid">; defm int_nvvm_read_ptx_sreg_ntid : PTXReadSRegIntrinsic_v4i32<"ntid">; @@ -4375,14 +4428,16 @@ foreach sync = [false, true] in { foreach return_pred = [false, true] in { foreach i = [SHFL_INFO<sync, mode, type, return_pred>] in { if i.withGccBuiltin then { - def i.Name : GCCBuiltin<i.Builtin>, + def i.Name : ClangBuiltin<i.Builtin>, Intrinsic<i.RetTy, i.ArgsTy, - [IntrInaccessibleMemOnly, IntrConvergent], + [IntrInaccessibleMemOnly, IntrConvergent, + IntrNoCallback], i.IntrName>; } if i.withoutGccBuiltin then { def i.Name : Intrinsic<i.RetTy, i.ArgsTy, - [IntrInaccessibleMemOnly, IntrConvergent], i.IntrName>; + [IntrInaccessibleMemOnly, IntrConvergent, + IntrNoCallback], i.IntrName>; } } } @@ -4397,23 +4452,23 @@ foreach sync = [false, true] in { // vote.all pred def int_nvvm_vote_all : Intrinsic<[llvm_i1_ty], [llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.all">, - GCCBuiltin<"__nvvm_vote_all">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.all">, + ClangBuiltin<"__nvvm_vote_all">; // vote.any pred def int_nvvm_vote_any : Intrinsic<[llvm_i1_ty], [llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.any">, - GCCBuiltin<"__nvvm_vote_any">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.any">, + ClangBuiltin<"__nvvm_vote_any">; // vote.uni pred def int_nvvm_vote_uni : Intrinsic<[llvm_i1_ty], [llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.uni">, - GCCBuiltin<"__nvvm_vote_uni">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.uni">, + ClangBuiltin<"__nvvm_vote_uni">; // vote.ballot pred def int_nvvm_vote_ballot : Intrinsic<[llvm_i32_ty], [llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.ballot">, - GCCBuiltin<"__nvvm_vote_ballot">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.ballot">, + ClangBuiltin<"__nvvm_vote_ballot">; // // VOTE.SYNC @@ -4422,23 +4477,23 @@ def int_nvvm_vote_ballot : // vote.sync.all mask, pred def int_nvvm_vote_all_sync : Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.all.sync">, - GCCBuiltin<"__nvvm_vote_all_sync">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.all.sync">, + ClangBuiltin<"__nvvm_vote_all_sync">; // vote.sync.any mask, pred def int_nvvm_vote_any_sync : Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.any.sync">, - GCCBuiltin<"__nvvm_vote_any_sync">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.any.sync">, + ClangBuiltin<"__nvvm_vote_any_sync">; // vote.sync.uni mask, pred def int_nvvm_vote_uni_sync : Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.uni.sync">, - GCCBuiltin<"__nvvm_vote_uni_sync">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.uni.sync">, + ClangBuiltin<"__nvvm_vote_uni_sync">; // vote.sync.ballot mask, pred def int_nvvm_vote_ballot_sync : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i1_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.ballot.sync">, - GCCBuiltin<"__nvvm_vote_ballot_sync">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.vote.ballot.sync">, + ClangBuiltin<"__nvvm_vote_ballot_sync">; // // MATCH.SYNC @@ -4446,13 +4501,13 @@ def int_nvvm_vote_ballot_sync : // match.any.sync.b32 mask, value def int_nvvm_match_any_sync_i32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.any.sync.i32">, - GCCBuiltin<"__nvvm_match_any_sync_i32">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.match.any.sync.i32">, + ClangBuiltin<"__nvvm_match_any_sync_i32">; // match.any.sync.b64 mask, value def int_nvvm_match_any_sync_i64 : - Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.any.sync.i64">, - GCCBuiltin<"__nvvm_match_any_sync_i64">; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.match.any.sync.i64">, + ClangBuiltin<"__nvvm_match_any_sync_i64">; // match.all instruction have two variants -- one returns a single value, another // returns a pair {value, predicate}. We currently only implement the latter as @@ -4461,54 +4516,54 @@ def int_nvvm_match_any_sync_i64 : // match.all.sync.b32p mask, value def int_nvvm_match_all_sync_i32p : Intrinsic<[llvm_i32_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.all.sync.i32p">; + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.match.all.sync.i32p">; // match.all.sync.b64p mask, value def int_nvvm_match_all_sync_i64p : - Intrinsic<[llvm_i64_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i64_ty], - [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.all.sync.i64p">; + Intrinsic<[llvm_i32_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback], "llvm.nvvm.match.all.sync.i64p">; // // REDUX.SYNC // // redux.sync.min.u32 dst, src, membermask; -def int_nvvm_redux_sync_umin : GCCBuiltin<"__nvvm_redux_sync_umin">, +def int_nvvm_redux_sync_umin : ClangBuiltin<"__nvvm_redux_sync_umin">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.max.u32 dst, src, membermask; -def int_nvvm_redux_sync_umax : GCCBuiltin<"__nvvm_redux_sync_umax">, +def int_nvvm_redux_sync_umax : ClangBuiltin<"__nvvm_redux_sync_umax">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.add.s32 dst, src, membermask; -def int_nvvm_redux_sync_add : GCCBuiltin<"__nvvm_redux_sync_add">, +def int_nvvm_redux_sync_add : ClangBuiltin<"__nvvm_redux_sync_add">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.min.s32 dst, src, membermask; -def int_nvvm_redux_sync_min : GCCBuiltin<"__nvvm_redux_sync_min">, +def int_nvvm_redux_sync_min : ClangBuiltin<"__nvvm_redux_sync_min">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.max.s32 dst, src, membermask; -def int_nvvm_redux_sync_max : GCCBuiltin<"__nvvm_redux_sync_max">, +def int_nvvm_redux_sync_max : ClangBuiltin<"__nvvm_redux_sync_max">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.and.b32 dst, src, membermask; -def int_nvvm_redux_sync_and : GCCBuiltin<"__nvvm_redux_sync_and">, +def int_nvvm_redux_sync_and : ClangBuiltin<"__nvvm_redux_sync_and">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.xor.b32 dst, src, membermask; -def int_nvvm_redux_sync_xor : GCCBuiltin<"__nvvm_redux_sync_xor">, +def int_nvvm_redux_sync_xor : ClangBuiltin<"__nvvm_redux_sync_xor">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // redux.sync.or.b32 dst, src, membermask; -def int_nvvm_redux_sync_or : GCCBuiltin<"__nvvm_redux_sync_or">, +def int_nvvm_redux_sync_or : ClangBuiltin<"__nvvm_redux_sync_or">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrConvergent, IntrInaccessibleMemOnly]>; + [IntrConvergent, IntrInaccessibleMemOnly, IntrNoCallback]>; // // WMMA instructions @@ -4517,7 +4572,7 @@ def int_nvvm_redux_sync_or : GCCBuiltin<"__nvvm_redux_sync_or">, class NVVM_WMMA_LD<WMMA_REGS Frag, string Layout, int WithStride> : Intrinsic<Frag.regs, !if(WithStride, [llvm_anyptr_ty, llvm_i32_ty], [llvm_anyptr_ty]), - [IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>; // WMMA.STORE.D @@ -4527,7 +4582,7 @@ class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> [llvm_anyptr_ty], Frag.regs, !if(WithStride, [llvm_i32_ty], [])), - [IntrWriteMem, IntrArgMemOnly, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], + [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr>; // Create all load/store variants @@ -4550,7 +4605,7 @@ class NVVM_WMMA_MMA<string ALayout, string BLayout, int Satfinite, string rnd, s WMMA_REGS C, WMMA_REGS D> : Intrinsic<D.regs, !listconcat(A.regs, B.regs, C.regs), - [IntrNoMem], + [IntrNoMem, IntrNoCallback], WMMA_NAME<ALayout, BLayout, Satfinite, rnd, b1op, A, B, C, D>.llvm>; foreach layout_a = ["row", "col"] in { @@ -4577,7 +4632,7 @@ class NVVM_MMA<string ALayout, string BLayout, int Satfinite, string b1op, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> : Intrinsic<D.regs, !listconcat(A.regs, B.regs, C.regs), - [IntrNoMem], + [IntrNoMem, IntrNoCallback], MMA_NAME<ALayout, BLayout, Satfinite, b1op, A, B, C, D>.llvm>; foreach layout_a = ["row", "col"] in { @@ -4598,7 +4653,7 @@ foreach layout_a = ["row", "col"] in { // LDMATRIX class NVVM_LDMATRIX<WMMA_REGS Frag, int Transposed> : Intrinsic<Frag.regs, [llvm_anyptr_ty], - [IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<0>>, + [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], LDMATRIX_NAME<Frag, Transposed>.intr>; diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td index b01fa10763b8..577122328dd2 100644 --- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td +++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td @@ -18,7 +18,7 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". // dcba/dcbf/dcbi/dcbst/dcbt/dcbz/dcbzl(PPC970) instructions. def int_ppc_dcba : Intrinsic<[], [llvm_ptr_ty], []>; - def int_ppc_dcbf : GCCBuiltin<"__builtin_dcbf">, + def int_ppc_dcbf : ClangBuiltin<"__builtin_dcbf">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>; def int_ppc_dcbfps : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>; def int_ppc_dcbstps : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>; @@ -30,136 +30,170 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". def int_ppc_dcbzl : Intrinsic<[], [llvm_ptr_ty], []>; // Get content from current FPSCR register - def int_ppc_readflm : GCCBuiltin<"__builtin_readflm">, + def int_ppc_readflm : ClangBuiltin<"__builtin_readflm">, Intrinsic<[llvm_double_ty], [], [IntrNoMerge, IntrHasSideEffects]>; // Set FPSCR register, and return previous content - def int_ppc_setflm : GCCBuiltin<"__builtin_setflm">, + def int_ppc_setflm : ClangBuiltin<"__builtin_setflm">, Intrinsic<[llvm_double_ty], [llvm_double_ty], [IntrHasSideEffects]>; // Intrinsics for [double]word extended forms of divide instructions - def int_ppc_divwe : GCCBuiltin<"__builtin_divwe">, + def int_ppc_divwe : ClangBuiltin<"__builtin_divwe">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_divweu : GCCBuiltin<"__builtin_divweu">, + def int_ppc_divweu : ClangBuiltin<"__builtin_divweu">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_divde : GCCBuiltin<"__builtin_divde">, + def int_ppc_divde : ClangBuiltin<"__builtin_divde">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_ppc_divdeu : GCCBuiltin<"__builtin_divdeu">, + def int_ppc_divdeu : ClangBuiltin<"__builtin_divdeu">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_ppc_unpack_longdouble : GCCBuiltin<"__builtin_unpack_longdouble">, + def int_ppc_unpack_longdouble : ClangBuiltin<"__builtin_unpack_longdouble">, Intrinsic<[llvm_double_ty], [llvm_ppcf128_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_pack_longdouble : GCCBuiltin<"__builtin_pack_longdouble">, + def int_ppc_pack_longdouble : ClangBuiltin<"__builtin_pack_longdouble">, Intrinsic<[llvm_ppcf128_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; // Generate a random number - def int_ppc_darn : GCCBuiltin<"__builtin_darn">, - Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>; - def int_ppc_darnraw : GCCBuiltin<"__builtin_darn_raw">, - Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>; - def int_ppc_darn32 : GCCBuiltin<"__builtin_darn_32">, - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; + def int_ppc_darn : ClangBuiltin<"__builtin_darn">, + Intrinsic<[llvm_i64_ty], [], + [IntrNoMerge, IntrHasSideEffects]>; + def int_ppc_darnraw : ClangBuiltin<"__builtin_darn_raw">, + Intrinsic<[llvm_i64_ty], [], + [IntrNoMerge, IntrHasSideEffects]>; + def int_ppc_darn32 : ClangBuiltin<"__builtin_darn_32">, + Intrinsic<[llvm_i32_ty], [], + [IntrNoMerge, IntrHasSideEffects]>; // Bit permute doubleword - def int_ppc_bpermd : GCCBuiltin<"__builtin_bpermd">, + def int_ppc_bpermd : ClangBuiltin<"__builtin_bpermd">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; // Parallel Bits Deposit/Extract Doubleword Builtins. def int_ppc_pdepd - : GCCBuiltin<"__builtin_pdepd">, + : ClangBuiltin<"__builtin_pdepd">, Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_pextd - : GCCBuiltin<"__builtin_pextd">, + : ClangBuiltin<"__builtin_pextd">, Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; // Centrifuge Doubleword Builtin. def int_ppc_cfuged - : GCCBuiltin<"__builtin_cfuged">, + : ClangBuiltin<"__builtin_cfuged">, Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; // Count Leading / Trailing Zeroes under bit Mask Builtins. def int_ppc_cntlzdm - : GCCBuiltin<"__builtin_cntlzdm">, + : ClangBuiltin<"__builtin_cntlzdm">, Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_cnttzdm - : GCCBuiltin<"__builtin_cnttzdm">, + : ClangBuiltin<"__builtin_cnttzdm">, Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_truncf128_round_to_odd - : GCCBuiltin<"__builtin_truncf128_round_to_odd">, + : ClangBuiltin<"__builtin_truncf128_round_to_odd">, Intrinsic <[llvm_double_ty], [llvm_f128_ty], [IntrNoMem]>; def int_ppc_sqrtf128_round_to_odd - : GCCBuiltin<"__builtin_sqrtf128_round_to_odd">, + : ClangBuiltin<"__builtin_sqrtf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty], [IntrNoMem]>; def int_ppc_addf128_round_to_odd - : GCCBuiltin<"__builtin_addf128_round_to_odd">, + : ClangBuiltin<"__builtin_addf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty,llvm_f128_ty], [IntrNoMem]>; def int_ppc_subf128_round_to_odd - : GCCBuiltin<"__builtin_subf128_round_to_odd">, + : ClangBuiltin<"__builtin_subf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty,llvm_f128_ty], [IntrNoMem]>; def int_ppc_mulf128_round_to_odd - : GCCBuiltin<"__builtin_mulf128_round_to_odd">, + : ClangBuiltin<"__builtin_mulf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty,llvm_f128_ty], [IntrNoMem]>; def int_ppc_divf128_round_to_odd - : GCCBuiltin<"__builtin_divf128_round_to_odd">, + : ClangBuiltin<"__builtin_divf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty,llvm_f128_ty], [IntrNoMem]>; def int_ppc_fmaf128_round_to_odd - : GCCBuiltin<"__builtin_fmaf128_round_to_odd">, + : ClangBuiltin<"__builtin_fmaf128_round_to_odd">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty,llvm_f128_ty,llvm_f128_ty], [IntrNoMem]>; def int_ppc_scalar_extract_expq - : GCCBuiltin<"__builtin_vsx_scalar_extract_expq">, + : ClangBuiltin<"__builtin_vsx_scalar_extract_expq">, Intrinsic <[llvm_i64_ty], [llvm_f128_ty], [IntrNoMem]>; def int_ppc_scalar_insert_exp_qp - : GCCBuiltin<"__builtin_vsx_scalar_insert_exp_qp">, + : ClangBuiltin<"__builtin_vsx_scalar_insert_exp_qp">, Intrinsic <[llvm_f128_ty], [llvm_f128_ty, llvm_i64_ty], [IntrNoMem]>; // Intrinsics defined to maintain XL compatibility def int_ppc_tdw - : GCCBuiltin<"__builtin_ppc_tdw">, + : ClangBuiltin<"__builtin_ppc_tdw">, Intrinsic <[], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; def int_ppc_tw - : GCCBuiltin<"__builtin_ppc_tw">, + : ClangBuiltin<"__builtin_ppc_tw">, Intrinsic <[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; def int_ppc_trapd - : GCCBuiltin<"__builtin_ppc_trapd">, + : ClangBuiltin<"__builtin_ppc_trapd">, Intrinsic <[], [llvm_i64_ty], []>; def int_ppc_trap - : GCCBuiltin<"__builtin_ppc_trap">, + : ClangBuiltin<"__builtin_ppc_trap">, Intrinsic <[], [llvm_i32_ty], []>; def int_ppc_fcfid - : GCCBuiltin<"__builtin_ppc_fcfid">, + : ClangBuiltin<"__builtin_ppc_fcfid">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fcfud - : GCCBuiltin<"__builtin_ppc_fcfud">, + : ClangBuiltin<"__builtin_ppc_fcfud">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctid - : GCCBuiltin<"__builtin_ppc_fctid">, + : ClangBuiltin<"__builtin_ppc_fctid">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctidz - : GCCBuiltin<"__builtin_ppc_fctidz">, + : ClangBuiltin<"__builtin_ppc_fctidz">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctiw - : GCCBuiltin<"__builtin_ppc_fctiw">, + : ClangBuiltin<"__builtin_ppc_fctiw">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctiwz - : GCCBuiltin<"__builtin_ppc_fctiwz">, + : ClangBuiltin<"__builtin_ppc_fctiwz">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctudz - : GCCBuiltin<"__builtin_ppc_fctudz">, + : ClangBuiltin<"__builtin_ppc_fctudz">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fctuwz - : GCCBuiltin<"__builtin_ppc_fctuwz">, + : ClangBuiltin<"__builtin_ppc_fctuwz">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; + + // XL compatible select functions + // TODO: Add llvm_f128_ty support. + def int_ppc_maxfe + : Intrinsic< + [llvm_ppcf128_ty], + [llvm_ppcf128_ty, llvm_ppcf128_ty, llvm_ppcf128_ty, llvm_vararg_ty], + [IntrNoMem]>; + def int_ppc_maxfl + : Intrinsic< + [llvm_double_ty], + [llvm_double_ty, llvm_double_ty, llvm_double_ty, llvm_vararg_ty], + [IntrNoMem]>; + def int_ppc_maxfs + : Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_vararg_ty], + [IntrNoMem]>; + def int_ppc_minfe + : Intrinsic< + [llvm_ppcf128_ty], + [llvm_ppcf128_ty, llvm_ppcf128_ty, llvm_ppcf128_ty, llvm_vararg_ty], + [IntrNoMem]>; + def int_ppc_minfl + : Intrinsic< + [llvm_double_ty], + [llvm_double_ty, llvm_double_ty, llvm_double_ty, llvm_vararg_ty], + [IntrNoMem]>; + def int_ppc_minfs + : Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_vararg_ty], + [IntrNoMem]>; } let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". @@ -167,14 +201,14 @@ let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". class PowerPC_Vec_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types, list<LLVMType> param_types, list<IntrinsicProperty> properties> - : GCCBuiltin<!strconcat("__builtin_altivec_", GCCIntSuffix)>, + : ClangBuiltin<!strconcat("__builtin_altivec_", GCCIntSuffix)>, Intrinsic<ret_types, param_types, properties>; /// PowerPC_VSX_Intrinsic - Base class for all VSX intrinsics. class PowerPC_VSX_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types, list<LLVMType> param_types, list<IntrinsicProperty> properties> - : GCCBuiltin<!strconcat("__builtin_vsx_", GCCIntSuffix)>, + : ClangBuiltin<!strconcat("__builtin_vsx_", GCCIntSuffix)>, Intrinsic<ret_types, param_types, properties>; } @@ -289,31 +323,31 @@ class PowerPC_VSX_Sca_DDD_Intrinsic<string GCCIntSuffix> let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". // Data Stream Control. - def int_ppc_altivec_dss : GCCBuiltin<"__builtin_altivec_dss">, + def int_ppc_altivec_dss : ClangBuiltin<"__builtin_altivec_dss">, Intrinsic<[], [llvm_i32_ty], []>; - def int_ppc_altivec_dssall : GCCBuiltin<"__builtin_altivec_dssall">, + def int_ppc_altivec_dssall : ClangBuiltin<"__builtin_altivec_dssall">, Intrinsic<[], [], []>; - def int_ppc_altivec_dst : GCCBuiltin<"__builtin_altivec_dst">, + def int_ppc_altivec_dst : ClangBuiltin<"__builtin_altivec_dst">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; - def int_ppc_altivec_dstt : GCCBuiltin<"__builtin_altivec_dstt">, + def int_ppc_altivec_dstt : ClangBuiltin<"__builtin_altivec_dstt">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; - def int_ppc_altivec_dstst : GCCBuiltin<"__builtin_altivec_dstst">, + def int_ppc_altivec_dstst : ClangBuiltin<"__builtin_altivec_dstst">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; - def int_ppc_altivec_dststt : GCCBuiltin<"__builtin_altivec_dststt">, + def int_ppc_altivec_dststt : ClangBuiltin<"__builtin_altivec_dststt">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; // VSCR access. - def int_ppc_altivec_mfvscr : GCCBuiltin<"__builtin_altivec_mfvscr">, + def int_ppc_altivec_mfvscr : ClangBuiltin<"__builtin_altivec_mfvscr">, Intrinsic<[llvm_v8i16_ty], [], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_mtvscr : GCCBuiltin<"__builtin_altivec_mtvscr">, + def int_ppc_altivec_mtvscr : ClangBuiltin<"__builtin_altivec_mtvscr">, Intrinsic<[], [llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; @@ -349,354 +383,354 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". [IntrWriteMem, IntrArgMemOnly]>; // Comparisons setting a vector. - def int_ppc_altivec_vcmpbfp : GCCBuiltin<"__builtin_altivec_vcmpbfp">, + def int_ppc_altivec_vcmpbfp : ClangBuiltin<"__builtin_altivec_vcmpbfp">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpeqfp : GCCBuiltin<"__builtin_altivec_vcmpeqfp">, + def int_ppc_altivec_vcmpeqfp : ClangBuiltin<"__builtin_altivec_vcmpeqfp">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgefp : GCCBuiltin<"__builtin_altivec_vcmpgefp">, + def int_ppc_altivec_vcmpgefp : ClangBuiltin<"__builtin_altivec_vcmpgefp">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtfp : GCCBuiltin<"__builtin_altivec_vcmpgtfp">, + def int_ppc_altivec_vcmpgtfp : ClangBuiltin<"__builtin_altivec_vcmpgtfp">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequd : GCCBuiltin<"__builtin_altivec_vcmpequd">, + def int_ppc_altivec_vcmpequd : ClangBuiltin<"__builtin_altivec_vcmpequd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsd : GCCBuiltin<"__builtin_altivec_vcmpgtsd">, + def int_ppc_altivec_vcmpgtsd : ClangBuiltin<"__builtin_altivec_vcmpgtsd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtud : GCCBuiltin<"__builtin_altivec_vcmpgtud">, + def int_ppc_altivec_vcmpgtud : ClangBuiltin<"__builtin_altivec_vcmpgtud">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequw : GCCBuiltin<"__builtin_altivec_vcmpequw">, + def int_ppc_altivec_vcmpequw : ClangBuiltin<"__builtin_altivec_vcmpequw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsw : GCCBuiltin<"__builtin_altivec_vcmpgtsw">, + def int_ppc_altivec_vcmpgtsw : ClangBuiltin<"__builtin_altivec_vcmpgtsw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuw : GCCBuiltin<"__builtin_altivec_vcmpgtuw">, + def int_ppc_altivec_vcmpgtuw : ClangBuiltin<"__builtin_altivec_vcmpgtuw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnew : GCCBuiltin<"__builtin_altivec_vcmpnew">, + def int_ppc_altivec_vcmpnew : ClangBuiltin<"__builtin_altivec_vcmpnew">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezw : GCCBuiltin<"__builtin_altivec_vcmpnezw">, + def int_ppc_altivec_vcmpnezw : ClangBuiltin<"__builtin_altivec_vcmpnezw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequh : GCCBuiltin<"__builtin_altivec_vcmpequh">, + def int_ppc_altivec_vcmpequh : ClangBuiltin<"__builtin_altivec_vcmpequh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsh : GCCBuiltin<"__builtin_altivec_vcmpgtsh">, + def int_ppc_altivec_vcmpgtsh : ClangBuiltin<"__builtin_altivec_vcmpgtsh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuh : GCCBuiltin<"__builtin_altivec_vcmpgtuh">, + def int_ppc_altivec_vcmpgtuh : ClangBuiltin<"__builtin_altivec_vcmpgtuh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpneh : GCCBuiltin<"__builtin_altivec_vcmpneh">, + def int_ppc_altivec_vcmpneh : ClangBuiltin<"__builtin_altivec_vcmpneh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezh : GCCBuiltin<"__builtin_altivec_vcmpnezh">, + def int_ppc_altivec_vcmpnezh : ClangBuiltin<"__builtin_altivec_vcmpnezh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequb : GCCBuiltin<"__builtin_altivec_vcmpequb">, + def int_ppc_altivec_vcmpequb : ClangBuiltin<"__builtin_altivec_vcmpequb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsb : GCCBuiltin<"__builtin_altivec_vcmpgtsb">, + def int_ppc_altivec_vcmpgtsb : ClangBuiltin<"__builtin_altivec_vcmpgtsb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtub : GCCBuiltin<"__builtin_altivec_vcmpgtub">, + def int_ppc_altivec_vcmpgtub : ClangBuiltin<"__builtin_altivec_vcmpgtub">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpneb : GCCBuiltin<"__builtin_altivec_vcmpneb">, + def int_ppc_altivec_vcmpneb : ClangBuiltin<"__builtin_altivec_vcmpneb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezb : GCCBuiltin<"__builtin_altivec_vcmpnezb">, + def int_ppc_altivec_vcmpnezb : ClangBuiltin<"__builtin_altivec_vcmpnezb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequq : GCCBuiltin<"__builtin_altivec_vcmpequq">, + def int_ppc_altivec_vcmpequq : ClangBuiltin<"__builtin_altivec_vcmpequq">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsq : GCCBuiltin<"__builtin_altivec_vcmpgtsq">, + def int_ppc_altivec_vcmpgtsq : ClangBuiltin<"__builtin_altivec_vcmpgtsq">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuq : GCCBuiltin<"__builtin_altivec_vcmpgtuq">, + def int_ppc_altivec_vcmpgtuq : ClangBuiltin<"__builtin_altivec_vcmpgtuq">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequq_p : GCCBuiltin<"__builtin_altivec_vcmpequq_p">, + def int_ppc_altivec_vcmpequq_p : ClangBuiltin<"__builtin_altivec_vcmpequq_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty,llvm_v1i128_ty,llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsq_p : GCCBuiltin<"__builtin_altivec_vcmpgtsq_p">, + def int_ppc_altivec_vcmpgtsq_p : ClangBuiltin<"__builtin_altivec_vcmpgtsq_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty,llvm_v1i128_ty,llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuq_p : GCCBuiltin<"__builtin_altivec_vcmpgtuq_p">, + def int_ppc_altivec_vcmpgtuq_p : ClangBuiltin<"__builtin_altivec_vcmpgtuq_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty,llvm_v1i128_ty,llvm_v1i128_ty], [IntrNoMem]>; // Predicate Comparisons. The first operand specifies interpretation of CR6. - def int_ppc_altivec_vcmpbfp_p : GCCBuiltin<"__builtin_altivec_vcmpbfp_p">, + def int_ppc_altivec_vcmpbfp_p : ClangBuiltin<"__builtin_altivec_vcmpbfp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpeqfp_p : GCCBuiltin<"__builtin_altivec_vcmpeqfp_p">, + def int_ppc_altivec_vcmpeqfp_p : ClangBuiltin<"__builtin_altivec_vcmpeqfp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgefp_p : GCCBuiltin<"__builtin_altivec_vcmpgefp_p">, + def int_ppc_altivec_vcmpgefp_p : ClangBuiltin<"__builtin_altivec_vcmpgefp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtfp_p : GCCBuiltin<"__builtin_altivec_vcmpgtfp_p">, + def int_ppc_altivec_vcmpgtfp_p : ClangBuiltin<"__builtin_altivec_vcmpgtfp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequd_p : GCCBuiltin<"__builtin_altivec_vcmpequd_p">, + def int_ppc_altivec_vcmpequd_p : ClangBuiltin<"__builtin_altivec_vcmpequd_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2i64_ty,llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsd_p : GCCBuiltin<"__builtin_altivec_vcmpgtsd_p">, + def int_ppc_altivec_vcmpgtsd_p : ClangBuiltin<"__builtin_altivec_vcmpgtsd_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2i64_ty,llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtud_p : GCCBuiltin<"__builtin_altivec_vcmpgtud_p">, + def int_ppc_altivec_vcmpgtud_p : ClangBuiltin<"__builtin_altivec_vcmpgtud_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2i64_ty,llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequw_p : GCCBuiltin<"__builtin_altivec_vcmpequw_p">, + def int_ppc_altivec_vcmpequw_p : ClangBuiltin<"__builtin_altivec_vcmpequw_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4i32_ty,llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsw_p : GCCBuiltin<"__builtin_altivec_vcmpgtsw_p">, + def int_ppc_altivec_vcmpgtsw_p : ClangBuiltin<"__builtin_altivec_vcmpgtsw_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4i32_ty,llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuw_p : GCCBuiltin<"__builtin_altivec_vcmpgtuw_p">, + def int_ppc_altivec_vcmpgtuw_p : ClangBuiltin<"__builtin_altivec_vcmpgtuw_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4i32_ty,llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnew_p : GCCBuiltin<"__builtin_altivec_vcmpnew_p">, + def int_ppc_altivec_vcmpnew_p : ClangBuiltin<"__builtin_altivec_vcmpnew_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4i32_ty,llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezw_p : GCCBuiltin<"__builtin_altivec_vcmpnezw_p">, + def int_ppc_altivec_vcmpnezw_p : ClangBuiltin<"__builtin_altivec_vcmpnezw_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4i32_ty,llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequh_p : GCCBuiltin<"__builtin_altivec_vcmpequh_p">, + def int_ppc_altivec_vcmpequh_p : ClangBuiltin<"__builtin_altivec_vcmpequh_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsh_p : GCCBuiltin<"__builtin_altivec_vcmpgtsh_p">, + def int_ppc_altivec_vcmpgtsh_p : ClangBuiltin<"__builtin_altivec_vcmpgtsh_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtuh_p : GCCBuiltin<"__builtin_altivec_vcmpgtuh_p">, + def int_ppc_altivec_vcmpgtuh_p : ClangBuiltin<"__builtin_altivec_vcmpgtuh_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpneh_p : GCCBuiltin<"__builtin_altivec_vcmpneh_p">, + def int_ppc_altivec_vcmpneh_p : ClangBuiltin<"__builtin_altivec_vcmpneh_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezh_p : GCCBuiltin<"__builtin_altivec_vcmpnezh_p">, + def int_ppc_altivec_vcmpnezh_p : ClangBuiltin<"__builtin_altivec_vcmpnezh_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpequb_p : GCCBuiltin<"__builtin_altivec_vcmpequb_p">, + def int_ppc_altivec_vcmpequb_p : ClangBuiltin<"__builtin_altivec_vcmpequb_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtsb_p : GCCBuiltin<"__builtin_altivec_vcmpgtsb_p">, + def int_ppc_altivec_vcmpgtsb_p : ClangBuiltin<"__builtin_altivec_vcmpgtsb_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpgtub_p : GCCBuiltin<"__builtin_altivec_vcmpgtub_p">, + def int_ppc_altivec_vcmpgtub_p : ClangBuiltin<"__builtin_altivec_vcmpgtub_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpneb_p : GCCBuiltin<"__builtin_altivec_vcmpneb_p">, + def int_ppc_altivec_vcmpneb_p : ClangBuiltin<"__builtin_altivec_vcmpneb_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vcmpnezb_p : GCCBuiltin<"__builtin_altivec_vcmpnezb_p">, + def int_ppc_altivec_vcmpnezb_p : ClangBuiltin<"__builtin_altivec_vcmpnezb_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vclzlsbb : GCCBuiltin<"__builtin_altivec_vclzlsbb">, + def int_ppc_altivec_vclzlsbb : ClangBuiltin<"__builtin_altivec_vclzlsbb">, Intrinsic<[llvm_i32_ty],[llvm_v16i8_ty],[IntrNoMem]>; - def int_ppc_altivec_vctzlsbb : GCCBuiltin<"__builtin_altivec_vctzlsbb">, + def int_ppc_altivec_vctzlsbb : ClangBuiltin<"__builtin_altivec_vctzlsbb">, Intrinsic<[llvm_i32_ty],[llvm_v16i8_ty],[IntrNoMem]>; - def int_ppc_altivec_vprtybw : GCCBuiltin<"__builtin_altivec_vprtybw">, + def int_ppc_altivec_vprtybw : ClangBuiltin<"__builtin_altivec_vprtybw">, Intrinsic<[llvm_v4i32_ty],[llvm_v4i32_ty],[IntrNoMem]>; - def int_ppc_altivec_vprtybd : GCCBuiltin<"__builtin_altivec_vprtybd">, + def int_ppc_altivec_vprtybd : ClangBuiltin<"__builtin_altivec_vprtybd">, Intrinsic<[llvm_v2i64_ty],[llvm_v2i64_ty],[IntrNoMem]>; - def int_ppc_altivec_vprtybq : GCCBuiltin<"__builtin_altivec_vprtybq">, + def int_ppc_altivec_vprtybq : ClangBuiltin<"__builtin_altivec_vprtybq">, Intrinsic<[llvm_v1i128_ty],[llvm_v1i128_ty],[IntrNoMem]>; // BCD intrinsics. - def int_ppc_bcdadd : GCCBuiltin<"__builtin_ppc_bcdadd">, Intrinsic< + def int_ppc_bcdadd : ClangBuiltin<"__builtin_ppc_bcdadd">, Intrinsic< [llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_ppc_bcdadd_p : GCCBuiltin<"__builtin_ppc_bcdadd_p">, Intrinsic< + def int_ppc_bcdadd_p : ClangBuiltin<"__builtin_ppc_bcdadd_p">, Intrinsic< [llvm_i32_ty], [llvm_i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; - def int_ppc_bcdsub : GCCBuiltin<"__builtin_ppc_bcdsub">, Intrinsic< + def int_ppc_bcdsub : ClangBuiltin<"__builtin_ppc_bcdsub">, Intrinsic< [llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_ppc_bcdsub_p : GCCBuiltin<"__builtin_ppc_bcdsub_p">, Intrinsic< + def int_ppc_bcdsub_p : ClangBuiltin<"__builtin_ppc_bcdsub_p">, Intrinsic< [llvm_i32_ty], [llvm_i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; // P10 Vector Extract with Mask - def int_ppc_altivec_vextractbm : GCCBuiltin<"__builtin_altivec_vextractbm">, + def int_ppc_altivec_vextractbm : ClangBuiltin<"__builtin_altivec_vextractbm">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vextracthm : GCCBuiltin<"__builtin_altivec_vextracthm">, + def int_ppc_altivec_vextracthm : ClangBuiltin<"__builtin_altivec_vextracthm">, Intrinsic<[llvm_i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vextractwm : GCCBuiltin<"__builtin_altivec_vextractwm">, + def int_ppc_altivec_vextractwm : ClangBuiltin<"__builtin_altivec_vextractwm">, Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextractdm : GCCBuiltin<"__builtin_altivec_vextractdm">, + def int_ppc_altivec_vextractdm : ClangBuiltin<"__builtin_altivec_vextractdm">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vextractqm : GCCBuiltin<"__builtin_altivec_vextractqm">, + def int_ppc_altivec_vextractqm : ClangBuiltin<"__builtin_altivec_vextractqm">, Intrinsic<[llvm_i32_ty], [llvm_v1i128_ty], [IntrNoMem]>; // P10 Vector Expand with Mask - def int_ppc_altivec_vexpandbm : GCCBuiltin<"__builtin_altivec_vexpandbm">, + def int_ppc_altivec_vexpandbm : ClangBuiltin<"__builtin_altivec_vexpandbm">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vexpandhm : GCCBuiltin<"__builtin_altivec_vexpandhm">, + def int_ppc_altivec_vexpandhm : ClangBuiltin<"__builtin_altivec_vexpandhm">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vexpandwm : GCCBuiltin<"__builtin_altivec_vexpandwm">, + def int_ppc_altivec_vexpandwm : ClangBuiltin<"__builtin_altivec_vexpandwm">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vexpanddm : GCCBuiltin<"__builtin_altivec_vexpanddm">, + def int_ppc_altivec_vexpanddm : ClangBuiltin<"__builtin_altivec_vexpanddm">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vexpandqm : GCCBuiltin<"__builtin_altivec_vexpandqm">, + def int_ppc_altivec_vexpandqm : ClangBuiltin<"__builtin_altivec_vexpandqm">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty], [IntrNoMem]>; // P10 Vector Count with Mask intrinsics. - def int_ppc_altivec_vcntmbb : GCCBuiltin<"__builtin_altivec_vcntmbb">, + def int_ppc_altivec_vcntmbb : ClangBuiltin<"__builtin_altivec_vcntmbb">, Intrinsic<[llvm_i64_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vcntmbh : GCCBuiltin<"__builtin_altivec_vcntmbh">, + def int_ppc_altivec_vcntmbh : ClangBuiltin<"__builtin_altivec_vcntmbh">, Intrinsic<[llvm_i64_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vcntmbw : GCCBuiltin<"__builtin_altivec_vcntmbw">, + def int_ppc_altivec_vcntmbw : ClangBuiltin<"__builtin_altivec_vcntmbw">, Intrinsic<[llvm_i64_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vcntmbd : GCCBuiltin<"__builtin_altivec_vcntmbd">, + def int_ppc_altivec_vcntmbd : ClangBuiltin<"__builtin_altivec_vcntmbd">, Intrinsic<[llvm_i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; // P10 Move to VSR with Mask Intrinsics. - def int_ppc_altivec_mtvsrbm : GCCBuiltin<"__builtin_altivec_mtvsrbm">, + def int_ppc_altivec_mtvsrbm : ClangBuiltin<"__builtin_altivec_mtvsrbm">, Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_mtvsrhm : GCCBuiltin<"__builtin_altivec_mtvsrhm">, + def int_ppc_altivec_mtvsrhm : ClangBuiltin<"__builtin_altivec_mtvsrhm">, Intrinsic<[llvm_v8i16_ty], [llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_mtvsrwm : GCCBuiltin<"__builtin_altivec_mtvsrwm">, + def int_ppc_altivec_mtvsrwm : ClangBuiltin<"__builtin_altivec_mtvsrwm">, Intrinsic<[llvm_v4i32_ty], [llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_mtvsrdm : GCCBuiltin<"__builtin_altivec_mtvsrdm">, + def int_ppc_altivec_mtvsrdm : ClangBuiltin<"__builtin_altivec_mtvsrdm">, Intrinsic<[llvm_v2i64_ty], [llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_mtvsrqm : GCCBuiltin<"__builtin_altivec_mtvsrqm">, + def int_ppc_altivec_mtvsrqm : ClangBuiltin<"__builtin_altivec_mtvsrqm">, Intrinsic<[llvm_v1i128_ty], [llvm_i64_ty], [IntrNoMem]>; // P10 Vector Parallel Bits Deposit/Extract Doubleword Builtins. - def int_ppc_altivec_vpdepd : GCCBuiltin<"__builtin_altivec_vpdepd">, + def int_ppc_altivec_vpdepd : ClangBuiltin<"__builtin_altivec_vpdepd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vpextd : GCCBuiltin<"__builtin_altivec_vpextd">, + def int_ppc_altivec_vpextd : ClangBuiltin<"__builtin_altivec_vpextd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; // P10 Vector String Isolate Intrinsics. - def int_ppc_altivec_vstribr : GCCBuiltin<"__builtin_altivec_vstribr">, + def int_ppc_altivec_vstribr : ClangBuiltin<"__builtin_altivec_vstribr">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vstribl : GCCBuiltin<"__builtin_altivec_vstribl">, + def int_ppc_altivec_vstribl : ClangBuiltin<"__builtin_altivec_vstribl">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vstrihr : GCCBuiltin<"__builtin_altivec_vstrihr">, + def int_ppc_altivec_vstrihr : ClangBuiltin<"__builtin_altivec_vstrihr">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vstrihl : GCCBuiltin<"__builtin_altivec_vstrihl">, + def int_ppc_altivec_vstrihl : ClangBuiltin<"__builtin_altivec_vstrihl">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; // Predicate Intrinsics: The first operand specifies interpretation of CR6. - def int_ppc_altivec_vstribr_p : GCCBuiltin<"__builtin_altivec_vstribr_p">, + def int_ppc_altivec_vstribr_p : ClangBuiltin<"__builtin_altivec_vstribr_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vstribl_p : GCCBuiltin<"__builtin_altivec_vstribl_p">, + def int_ppc_altivec_vstribl_p : ClangBuiltin<"__builtin_altivec_vstribl_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vstrihr_p : GCCBuiltin<"__builtin_altivec_vstrihr_p">, + def int_ppc_altivec_vstrihr_p : ClangBuiltin<"__builtin_altivec_vstrihr_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vstrihl_p : GCCBuiltin<"__builtin_altivec_vstrihl_p">, + def int_ppc_altivec_vstrihl_p : ClangBuiltin<"__builtin_altivec_vstrihl_p">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_v8i16_ty], [IntrNoMem]>; // P10 Vector Centrifuge Builtin. - def int_ppc_altivec_vcfuged : GCCBuiltin<"__builtin_altivec_vcfuged">, + def int_ppc_altivec_vcfuged : ClangBuiltin<"__builtin_altivec_vcfuged">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; // P10 Vector Gather Every Nth Bit Builtin. - def int_ppc_altivec_vgnb : GCCBuiltin<"__builtin_altivec_vgnb">, + def int_ppc_altivec_vgnb : ClangBuiltin<"__builtin_altivec_vgnb">, Intrinsic<[llvm_i64_ty], [llvm_v1i128_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; // P10 Vector Clear Bytes - def int_ppc_altivec_vclrlb : GCCBuiltin<"__builtin_altivec_vclrlb">, + def int_ppc_altivec_vclrlb : ClangBuiltin<"__builtin_altivec_vclrlb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vclrrb : GCCBuiltin<"__builtin_altivec_vclrrb">, + def int_ppc_altivec_vclrrb : ClangBuiltin<"__builtin_altivec_vclrrb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; // P10 Vector Shift Double Bit Immediate. - def int_ppc_altivec_vsldbi : GCCBuiltin<"__builtin_altivec_vsldbi">, + def int_ppc_altivec_vsldbi : ClangBuiltin<"__builtin_altivec_vsldbi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_ppc_altivec_vsrdbi : GCCBuiltin<"__builtin_altivec_vsrdbi">, + def int_ppc_altivec_vsrdbi : ClangBuiltin<"__builtin_altivec_vsrdbi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; // P10 Vector Insert. - def int_ppc_altivec_vinsblx : GCCBuiltin<"__builtin_altivec_vinsblx">, + def int_ppc_altivec_vinsblx : ClangBuiltin<"__builtin_altivec_vinsblx">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinsbrx : GCCBuiltin<"__builtin_altivec_vinsbrx">, + def int_ppc_altivec_vinsbrx : ClangBuiltin<"__builtin_altivec_vinsbrx">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinshlx : GCCBuiltin<"__builtin_altivec_vinshlx">, + def int_ppc_altivec_vinshlx : ClangBuiltin<"__builtin_altivec_vinshlx">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinshrx : GCCBuiltin<"__builtin_altivec_vinshrx">, + def int_ppc_altivec_vinshrx : ClangBuiltin<"__builtin_altivec_vinshrx">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinswlx : GCCBuiltin<"__builtin_altivec_vinswlx">, + def int_ppc_altivec_vinswlx : ClangBuiltin<"__builtin_altivec_vinswlx">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinswrx : GCCBuiltin<"__builtin_altivec_vinswrx">, + def int_ppc_altivec_vinswrx : ClangBuiltin<"__builtin_altivec_vinswrx">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinsdlx : GCCBuiltin<"__builtin_altivec_vinsdlx">, + def int_ppc_altivec_vinsdlx : ClangBuiltin<"__builtin_altivec_vinsdlx">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vinsdrx : GCCBuiltin<"__builtin_altivec_vinsdrx">, + def int_ppc_altivec_vinsdrx : ClangBuiltin<"__builtin_altivec_vinsdrx">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vinsbvlx : GCCBuiltin<"__builtin_altivec_vinsbvlx">, + def int_ppc_altivec_vinsbvlx : ClangBuiltin<"__builtin_altivec_vinsbvlx">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vinsbvrx : GCCBuiltin<"__builtin_altivec_vinsbvrx">, + def int_ppc_altivec_vinsbvrx : ClangBuiltin<"__builtin_altivec_vinsbvrx">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vinshvlx : GCCBuiltin<"__builtin_altivec_vinshvlx">, + def int_ppc_altivec_vinshvlx : ClangBuiltin<"__builtin_altivec_vinshvlx">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vinshvrx : GCCBuiltin<"__builtin_altivec_vinshvrx">, + def int_ppc_altivec_vinshvrx : ClangBuiltin<"__builtin_altivec_vinshvrx">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vinswvlx : GCCBuiltin<"__builtin_altivec_vinswvlx">, + def int_ppc_altivec_vinswvlx : ClangBuiltin<"__builtin_altivec_vinswvlx">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vinswvrx : GCCBuiltin<"__builtin_altivec_vinswvrx">, + def int_ppc_altivec_vinswvrx : ClangBuiltin<"__builtin_altivec_vinswvrx">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], [IntrNoMem]>; @@ -710,35 +744,35 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". [llvm_v2i64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; // P10 Vector Extract. - def int_ppc_altivec_vextdubvlx : GCCBuiltin<"__builtin_altivec_vextdubvlx">, + def int_ppc_altivec_vextdubvlx : ClangBuiltin<"__builtin_altivec_vextdubvlx">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextdubvrx : GCCBuiltin<"__builtin_altivec_vextdubvrx">, + def int_ppc_altivec_vextdubvrx : ClangBuiltin<"__builtin_altivec_vextdubvrx">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextduhvlx : GCCBuiltin<"__builtin_altivec_vextduhvlx">, + def int_ppc_altivec_vextduhvlx : ClangBuiltin<"__builtin_altivec_vextduhvlx">, Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextduhvrx : GCCBuiltin<"__builtin_altivec_vextduhvrx">, + def int_ppc_altivec_vextduhvrx : ClangBuiltin<"__builtin_altivec_vextduhvrx">, Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextduwvlx : GCCBuiltin<"__builtin_altivec_vextduwvlx">, + def int_ppc_altivec_vextduwvlx : ClangBuiltin<"__builtin_altivec_vextduwvlx">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextduwvrx : GCCBuiltin<"__builtin_altivec_vextduwvrx">, + def int_ppc_altivec_vextduwvrx : ClangBuiltin<"__builtin_altivec_vextduwvrx">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextddvlx : GCCBuiltin<"__builtin_altivec_vextddvlx">, + def int_ppc_altivec_vextddvlx : ClangBuiltin<"__builtin_altivec_vextddvlx">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextddvrx : GCCBuiltin<"__builtin_altivec_vextddvrx">, + def int_ppc_altivec_vextddvrx : ClangBuiltin<"__builtin_altivec_vextddvrx">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; @@ -796,229 +830,229 @@ def int_ppc_altivec_vsubcuq : PowerPC_Vec_QQQ_Intrinsic<"vsubcuq">; let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". // Saturating multiply-adds. - def int_ppc_altivec_vmhaddshs : GCCBuiltin<"__builtin_altivec_vmhaddshs">, + def int_ppc_altivec_vmhaddshs : ClangBuiltin<"__builtin_altivec_vmhaddshs">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vmhraddshs : GCCBuiltin<"__builtin_altivec_vmhraddshs">, + def int_ppc_altivec_vmhraddshs : ClangBuiltin<"__builtin_altivec_vmhraddshs">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vmaddfp : GCCBuiltin<"__builtin_altivec_vmaddfp">, + def int_ppc_altivec_vmaddfp : ClangBuiltin<"__builtin_altivec_vmaddfp">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vnmsubfp : GCCBuiltin<"__builtin_altivec_vnmsubfp">, + def int_ppc_altivec_vnmsubfp : ClangBuiltin<"__builtin_altivec_vnmsubfp">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; // Vector Multiply Sum Instructions. - def int_ppc_altivec_vmsummbm : GCCBuiltin<"__builtin_altivec_vmsummbm">, + def int_ppc_altivec_vmsummbm : ClangBuiltin<"__builtin_altivec_vmsummbm">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vmsumshm : GCCBuiltin<"__builtin_altivec_vmsumshm">, + def int_ppc_altivec_vmsumshm : ClangBuiltin<"__builtin_altivec_vmsumshm">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vmsumshs : GCCBuiltin<"__builtin_altivec_vmsumshs">, + def int_ppc_altivec_vmsumshs : ClangBuiltin<"__builtin_altivec_vmsumshs">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vmsumubm : GCCBuiltin<"__builtin_altivec_vmsumubm">, + def int_ppc_altivec_vmsumubm : ClangBuiltin<"__builtin_altivec_vmsumubm">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vmsumuhm : GCCBuiltin<"__builtin_altivec_vmsumuhm">, + def int_ppc_altivec_vmsumuhm : ClangBuiltin<"__builtin_altivec_vmsumuhm">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vmsumudm : GCCBuiltin<"__builtin_altivec_vmsumudm">, + def int_ppc_altivec_vmsumudm : ClangBuiltin<"__builtin_altivec_vmsumudm">, Intrinsic<[llvm_v1i128_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vmsumuhs : GCCBuiltin<"__builtin_altivec_vmsumuhs">, + def int_ppc_altivec_vmsumuhs : ClangBuiltin<"__builtin_altivec_vmsumuhs">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vmsumcud : GCCBuiltin<"__builtin_altivec_vmsumcud">, + def int_ppc_altivec_vmsumcud : ClangBuiltin<"__builtin_altivec_vmsumcud">, Intrinsic<[llvm_v1i128_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v1i128_ty], [IntrNoMem]>; // Vector Multiply Instructions. - def int_ppc_altivec_vmulesb : GCCBuiltin<"__builtin_altivec_vmulesb">, + def int_ppc_altivec_vmulesb : ClangBuiltin<"__builtin_altivec_vmulesb">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulesh : GCCBuiltin<"__builtin_altivec_vmulesh">, + def int_ppc_altivec_vmulesh : ClangBuiltin<"__builtin_altivec_vmulesh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulesw : GCCBuiltin<"__builtin_altivec_vmulesw">, + def int_ppc_altivec_vmulesw : ClangBuiltin<"__builtin_altivec_vmulesw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_ppc_altivec_vmulesd : PowerPC_Vec_QDD_Intrinsic<"vmulesd">; - def int_ppc_altivec_vmuleub : GCCBuiltin<"__builtin_altivec_vmuleub">, + def int_ppc_altivec_vmuleub : ClangBuiltin<"__builtin_altivec_vmuleub">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vmuleuh : GCCBuiltin<"__builtin_altivec_vmuleuh">, + def int_ppc_altivec_vmuleuh : ClangBuiltin<"__builtin_altivec_vmuleuh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vmuleuw : GCCBuiltin<"__builtin_altivec_vmuleuw">, + def int_ppc_altivec_vmuleuw : ClangBuiltin<"__builtin_altivec_vmuleuw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_ppc_altivec_vmuleud : PowerPC_Vec_QDD_Intrinsic<"vmuleud">; - def int_ppc_altivec_vmulosb : GCCBuiltin<"__builtin_altivec_vmulosb">, + def int_ppc_altivec_vmulosb : ClangBuiltin<"__builtin_altivec_vmulosb">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulosh : GCCBuiltin<"__builtin_altivec_vmulosh">, + def int_ppc_altivec_vmulosh : ClangBuiltin<"__builtin_altivec_vmulosh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulosw : GCCBuiltin<"__builtin_altivec_vmulosw">, + def int_ppc_altivec_vmulosw : ClangBuiltin<"__builtin_altivec_vmulosw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_ppc_altivec_vmulosd : PowerPC_Vec_QDD_Intrinsic<"vmulosd">; - def int_ppc_altivec_vmuloub : GCCBuiltin<"__builtin_altivec_vmuloub">, + def int_ppc_altivec_vmuloub : ClangBuiltin<"__builtin_altivec_vmuloub">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulouh : GCCBuiltin<"__builtin_altivec_vmulouh">, + def int_ppc_altivec_vmulouh : ClangBuiltin<"__builtin_altivec_vmulouh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vmulouw : GCCBuiltin<"__builtin_altivec_vmulouw">, + def int_ppc_altivec_vmulouw : ClangBuiltin<"__builtin_altivec_vmulouw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_ppc_altivec_vmuloud : PowerPC_Vec_QDD_Intrinsic<"vmuloud">; // Vector Sum Instructions. - def int_ppc_altivec_vsumsws : GCCBuiltin<"__builtin_altivec_vsumsws">, + def int_ppc_altivec_vsumsws : ClangBuiltin<"__builtin_altivec_vsumsws">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vsum2sws : GCCBuiltin<"__builtin_altivec_vsum2sws">, + def int_ppc_altivec_vsum2sws : ClangBuiltin<"__builtin_altivec_vsum2sws">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vsum4sbs : GCCBuiltin<"__builtin_altivec_vsum4sbs">, + def int_ppc_altivec_vsum4sbs : ClangBuiltin<"__builtin_altivec_vsum4sbs">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vsum4shs : GCCBuiltin<"__builtin_altivec_vsum4shs">, + def int_ppc_altivec_vsum4shs : ClangBuiltin<"__builtin_altivec_vsum4shs">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vsum4ubs : GCCBuiltin<"__builtin_altivec_vsum4ubs">, + def int_ppc_altivec_vsum4ubs : ClangBuiltin<"__builtin_altivec_vsum4ubs">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; // Vector Sign Extension Instructions - def int_ppc_altivec_vextsb2w : GCCBuiltin<"__builtin_altivec_vextsb2w">, + def int_ppc_altivec_vextsb2w : ClangBuiltin<"__builtin_altivec_vextsb2w">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vextsb2d : GCCBuiltin<"__builtin_altivec_vextsb2d">, + def int_ppc_altivec_vextsb2d : ClangBuiltin<"__builtin_altivec_vextsb2d">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vextsh2w : GCCBuiltin<"__builtin_altivec_vextsh2w">, + def int_ppc_altivec_vextsh2w : ClangBuiltin<"__builtin_altivec_vextsh2w">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vextsh2d : GCCBuiltin<"__builtin_altivec_vextsh2d">, + def int_ppc_altivec_vextsh2d : ClangBuiltin<"__builtin_altivec_vextsh2d">, Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vextsw2d : GCCBuiltin<"__builtin_altivec_vextsw2d">, + def int_ppc_altivec_vextsw2d : ClangBuiltin<"__builtin_altivec_vextsw2d">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vextsd2q : GCCBuiltin<"__builtin_altivec_vextsd2q">, + def int_ppc_altivec_vextsd2q : ClangBuiltin<"__builtin_altivec_vextsd2q">, Intrinsic<[llvm_v1i128_ty], [llvm_v2i64_ty], [IntrNoMem]>; // Other multiplies. - def int_ppc_altivec_vmladduhm : GCCBuiltin<"__builtin_altivec_vmladduhm">, + def int_ppc_altivec_vmladduhm : ClangBuiltin<"__builtin_altivec_vmladduhm">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; // Packs. - def int_ppc_altivec_vpkpx : GCCBuiltin<"__builtin_altivec_vpkpx">, + def int_ppc_altivec_vpkpx : ClangBuiltin<"__builtin_altivec_vpkpx">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vpkshss : GCCBuiltin<"__builtin_altivec_vpkshss">, + def int_ppc_altivec_vpkshss : ClangBuiltin<"__builtin_altivec_vpkshss">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vpkshus : GCCBuiltin<"__builtin_altivec_vpkshus">, + def int_ppc_altivec_vpkshus : ClangBuiltin<"__builtin_altivec_vpkshus">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vpkswss : GCCBuiltin<"__builtin_altivec_vpkswss">, + def int_ppc_altivec_vpkswss : ClangBuiltin<"__builtin_altivec_vpkswss">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vpkswus : GCCBuiltin<"__builtin_altivec_vpkswus">, + def int_ppc_altivec_vpkswus : ClangBuiltin<"__builtin_altivec_vpkswus">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vpksdss : GCCBuiltin<"__builtin_altivec_vpksdss">, + def int_ppc_altivec_vpksdss : ClangBuiltin<"__builtin_altivec_vpksdss">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem, IntrHasSideEffects]>; - def int_ppc_altivec_vpksdus : GCCBuiltin<"__builtin_altivec_vpksdus">, + def int_ppc_altivec_vpksdus : ClangBuiltin<"__builtin_altivec_vpksdus">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem, IntrHasSideEffects]>; // vpkuhum is lowered to a shuffle. - def int_ppc_altivec_vpkuhus : GCCBuiltin<"__builtin_altivec_vpkuhus">, + def int_ppc_altivec_vpkuhus : ClangBuiltin<"__builtin_altivec_vpkuhus">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, IntrHasSideEffects]>; // vpkuwum is lowered to a shuffle. - def int_ppc_altivec_vpkuwus : GCCBuiltin<"__builtin_altivec_vpkuwus">, + def int_ppc_altivec_vpkuwus : ClangBuiltin<"__builtin_altivec_vpkuwus">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, IntrHasSideEffects]>; // vpkudum is lowered to a shuffle. - def int_ppc_altivec_vpkudus : GCCBuiltin<"__builtin_altivec_vpkudus">, + def int_ppc_altivec_vpkudus : ClangBuiltin<"__builtin_altivec_vpkudus">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem, IntrHasSideEffects]>; // Unpacks. - def int_ppc_altivec_vupkhpx : GCCBuiltin<"__builtin_altivec_vupkhpx">, + def int_ppc_altivec_vupkhpx : ClangBuiltin<"__builtin_altivec_vupkhpx">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vupkhsb : GCCBuiltin<"__builtin_altivec_vupkhsb">, + def int_ppc_altivec_vupkhsb : ClangBuiltin<"__builtin_altivec_vupkhsb">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vupkhsh : GCCBuiltin<"__builtin_altivec_vupkhsh">, + def int_ppc_altivec_vupkhsh : ClangBuiltin<"__builtin_altivec_vupkhsh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vupkhsw : GCCBuiltin<"__builtin_altivec_vupkhsw">, + def int_ppc_altivec_vupkhsw : ClangBuiltin<"__builtin_altivec_vupkhsw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vupklpx : GCCBuiltin<"__builtin_altivec_vupklpx">, + def int_ppc_altivec_vupklpx : ClangBuiltin<"__builtin_altivec_vupklpx">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vupklsb : GCCBuiltin<"__builtin_altivec_vupklsb">, + def int_ppc_altivec_vupklsb : ClangBuiltin<"__builtin_altivec_vupklsb">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vupklsh : GCCBuiltin<"__builtin_altivec_vupklsh">, + def int_ppc_altivec_vupklsh : ClangBuiltin<"__builtin_altivec_vupklsh">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_ppc_altivec_vupklsw : GCCBuiltin<"__builtin_altivec_vupklsw">, + def int_ppc_altivec_vupklsw : ClangBuiltin<"__builtin_altivec_vupklsw">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; // FP <-> integer conversion. - def int_ppc_altivec_vcfsx : GCCBuiltin<"__builtin_altivec_vcfsx">, + def int_ppc_altivec_vcfsx : ClangBuiltin<"__builtin_altivec_vcfsx">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vcfux : GCCBuiltin<"__builtin_altivec_vcfux">, + def int_ppc_altivec_vcfux : ClangBuiltin<"__builtin_altivec_vcfux">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vctsxs : GCCBuiltin<"__builtin_altivec_vctsxs">, + def int_ppc_altivec_vctsxs : ClangBuiltin<"__builtin_altivec_vctsxs">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vctuxs : GCCBuiltin<"__builtin_altivec_vctuxs">, + def int_ppc_altivec_vctuxs : ClangBuiltin<"__builtin_altivec_vctuxs">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_ppc_altivec_vrfim : GCCBuiltin<"__builtin_altivec_vrfim">, + def int_ppc_altivec_vrfim : ClangBuiltin<"__builtin_altivec_vrfim">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vrfin : GCCBuiltin<"__builtin_altivec_vrfin">, + def int_ppc_altivec_vrfin : ClangBuiltin<"__builtin_altivec_vrfin">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vrfip : GCCBuiltin<"__builtin_altivec_vrfip">, + def int_ppc_altivec_vrfip : ClangBuiltin<"__builtin_altivec_vrfip">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_ppc_altivec_vrfiz : GCCBuiltin<"__builtin_altivec_vrfiz">, + def int_ppc_altivec_vrfiz : ClangBuiltin<"__builtin_altivec_vrfiz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; // Add Extended Quadword - def int_ppc_altivec_vaddeuqm : GCCBuiltin<"__builtin_altivec_vaddeuqm">, + def int_ppc_altivec_vaddeuqm : ClangBuiltin<"__builtin_altivec_vaddeuqm">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vaddecuq : GCCBuiltin<"__builtin_altivec_vaddecuq">, + def int_ppc_altivec_vaddecuq : ClangBuiltin<"__builtin_altivec_vaddecuq">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; // Sub Extended Quadword - def int_ppc_altivec_vsubeuqm : GCCBuiltin<"__builtin_altivec_vsubeuqm">, + def int_ppc_altivec_vsubeuqm : ClangBuiltin<"__builtin_altivec_vsubeuqm">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; - def int_ppc_altivec_vsubecuq : GCCBuiltin<"__builtin_altivec_vsubecuq">, + def int_ppc_altivec_vsubecuq : ClangBuiltin<"__builtin_altivec_vsubecuq">, Intrinsic<[llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty], [IntrNoMem]>; // P10 Vector Count Leading / Trailing Zeroes under bit Mask Builtins. - def int_ppc_altivec_vclzdm : GCCBuiltin<"__builtin_altivec_vclzdm">, + def int_ppc_altivec_vclzdm : ClangBuiltin<"__builtin_altivec_vclzdm">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_ppc_altivec_vctzdm : GCCBuiltin<"__builtin_altivec_vctzdm">, + def int_ppc_altivec_vctzdm : ClangBuiltin<"__builtin_altivec_vctzdm">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; } @@ -1056,18 +1090,18 @@ let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". def int_ppc_altivec_lvsr : Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty], [IntrNoMem]>; - def int_ppc_altivec_vperm : GCCBuiltin<"__builtin_altivec_vperm_4si">, + def int_ppc_altivec_vperm : ClangBuiltin<"__builtin_altivec_vperm_4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vsel : GCCBuiltin<"__builtin_altivec_vsel_4si">, + def int_ppc_altivec_vsel : ClangBuiltin<"__builtin_altivec_vsel_4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_ppc_altivec_vgbbd : GCCBuiltin<"__builtin_altivec_vgbbd">, + def int_ppc_altivec_vgbbd : ClangBuiltin<"__builtin_altivec_vgbbd">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vbpermq : GCCBuiltin<"__builtin_altivec_vbpermq">, + def int_ppc_altivec_vbpermq : ClangBuiltin<"__builtin_altivec_vbpermq">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_ppc_altivec_vbpermd : GCCBuiltin<"__builtin_altivec_vbpermd">, + def int_ppc_altivec_vbpermd : ClangBuiltin<"__builtin_altivec_vbpermd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v16i8_ty], [IntrNoMem]>; } @@ -1081,23 +1115,23 @@ def int_ppc_altivec_vrsqrtefp : PowerPC_Vec_FF_Intrinsic<"vrsqrtefp">; // Crypto let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". def int_ppc_altivec_crypto_vsbox : - GCCBuiltin<"__builtin_altivec_crypto_vsbox">, + ClangBuiltin<"__builtin_altivec_crypto_vsbox">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_ppc_altivec_crypto_vpermxor : - GCCBuiltin<"__builtin_altivec_crypto_vpermxor">, + ClangBuiltin<"__builtin_altivec_crypto_vpermxor">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_ppc_altivec_crypto_vpermxor_be : - GCCBuiltin<"__builtin_altivec_crypto_vpermxor_be">, + ClangBuiltin<"__builtin_altivec_crypto_vpermxor_be">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_ppc_altivec_crypto_vshasigmad : - GCCBuiltin<"__builtin_altivec_crypto_vshasigmad">, + ClangBuiltin<"__builtin_altivec_crypto_vshasigmad">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; def int_ppc_altivec_crypto_vshasigmaw : - GCCBuiltin<"__builtin_altivec_crypto_vshasigmaw">, + ClangBuiltin<"__builtin_altivec_crypto_vshasigmaw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; } @@ -1224,52 +1258,52 @@ def int_ppc_vsx_xvrdpip : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; // Vector reciprocal estimate -def int_ppc_vsx_xvresp : GCCBuiltin<"__builtin_vsx_xvresp">, +def int_ppc_vsx_xvresp : ClangBuiltin<"__builtin_vsx_xvresp">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_ppc_vsx_xvredp : GCCBuiltin<"__builtin_vsx_xvredp">, +def int_ppc_vsx_xvredp : ClangBuiltin<"__builtin_vsx_xvredp">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; // Vector rsqrte -def int_ppc_vsx_xvrsqrtesp : GCCBuiltin<"__builtin_vsx_xvrsqrtesp">, +def int_ppc_vsx_xvrsqrtesp : ClangBuiltin<"__builtin_vsx_xvrsqrtesp">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; -def int_ppc_vsx_xvrsqrtedp : GCCBuiltin<"__builtin_vsx_xvrsqrtedp">, +def int_ppc_vsx_xvrsqrtedp : ClangBuiltin<"__builtin_vsx_xvrsqrtedp">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; // Vector compare def int_ppc_vsx_xvcmpeqdp : PowerPC_VSX_Intrinsic<"xvcmpeqdp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpeqdp_p : GCCBuiltin<"__builtin_vsx_xvcmpeqdp_p">, +def int_ppc_vsx_xvcmpeqdp_p : ClangBuiltin<"__builtin_vsx_xvcmpeqdp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], [IntrNoMem]>; def int_ppc_vsx_xvcmpeqsp : PowerPC_VSX_Intrinsic<"xvcmpeqsp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpeqsp_p : GCCBuiltin<"__builtin_vsx_xvcmpeqsp_p">, +def int_ppc_vsx_xvcmpeqsp_p : ClangBuiltin<"__builtin_vsx_xvcmpeqsp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; def int_ppc_vsx_xvcmpgedp : PowerPC_VSX_Intrinsic<"xvcmpgedp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpgedp_p : GCCBuiltin<"__builtin_vsx_xvcmpgedp_p">, +def int_ppc_vsx_xvcmpgedp_p : ClangBuiltin<"__builtin_vsx_xvcmpgedp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], [IntrNoMem]>; def int_ppc_vsx_xvcmpgesp : PowerPC_VSX_Intrinsic<"xvcmpgesp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpgesp_p : GCCBuiltin<"__builtin_vsx_xvcmpgesp_p">, +def int_ppc_vsx_xvcmpgesp_p : ClangBuiltin<"__builtin_vsx_xvcmpgesp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; def int_ppc_vsx_xvcmpgtdp : PowerPC_VSX_Intrinsic<"xvcmpgtdp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpgtdp_p : GCCBuiltin<"__builtin_vsx_xvcmpgtdp_p">, +def int_ppc_vsx_xvcmpgtdp_p : ClangBuiltin<"__builtin_vsx_xvcmpgtdp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], [IntrNoMem]>; def int_ppc_vsx_xvcmpgtsp : PowerPC_VSX_Intrinsic<"xvcmpgtsp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; -def int_ppc_vsx_xvcmpgtsp_p : GCCBuiltin<"__builtin_vsx_xvcmpgtsp_p">, +def int_ppc_vsx_xvcmpgtsp_p : ClangBuiltin<"__builtin_vsx_xvcmpgtsp_p">, Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; def int_ppc_vsx_xxleqv : @@ -1381,21 +1415,21 @@ def int_ppc_vsx_xxgenpcvdm : // P10 VSX Vector permute extended. def int_ppc_vsx_xxpermx : - GCCBuiltin<"__builtin_vsx_xxpermx">, + ClangBuiltin<"__builtin_vsx_xxpermx">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,llvm_v16i8_ty,llvm_v16i8_ty,llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; // P10 VSX Vector Blend Variable. -def int_ppc_vsx_xxblendvb: GCCBuiltin<"__builtin_vsx_xxblendvb">, +def int_ppc_vsx_xxblendvb: ClangBuiltin<"__builtin_vsx_xxblendvb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; -def int_ppc_vsx_xxblendvh: GCCBuiltin<"__builtin_vsx_xxblendvh">, +def int_ppc_vsx_xxblendvh: ClangBuiltin<"__builtin_vsx_xxblendvh">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,llvm_v8i16_ty], [IntrNoMem]>; -def int_ppc_vsx_xxblendvw: GCCBuiltin<"__builtin_vsx_xxblendvw">, +def int_ppc_vsx_xxblendvw: ClangBuiltin<"__builtin_vsx_xxblendvw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; -def int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">, +def int_ppc_vsx_xxblendvd: ClangBuiltin<"__builtin_vsx_xxblendvd">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; } @@ -1405,64 +1439,68 @@ def int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">, let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". -def int_ppc_tbegin : GCCBuiltin<"__builtin_tbegin">, +def int_ppc_tbegin : ClangBuiltin<"__builtin_tbegin">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; -def int_ppc_tend : GCCBuiltin<"__builtin_tend">, +def int_ppc_tend : ClangBuiltin<"__builtin_tend">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; -def int_ppc_tabort : GCCBuiltin<"__builtin_tabort">, +def int_ppc_tabort : ClangBuiltin<"__builtin_tabort">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; -def int_ppc_tabortwc : GCCBuiltin<"__builtin_tabortwc">, +def int_ppc_tabortwc : ClangBuiltin<"__builtin_tabortwc">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; -def int_ppc_tabortwci : GCCBuiltin<"__builtin_tabortwci">, +def int_ppc_tabortwci : ClangBuiltin<"__builtin_tabortwci">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; -def int_ppc_tabortdc : GCCBuiltin<"__builtin_tabortdc">, +def int_ppc_tabortdc : ClangBuiltin<"__builtin_tabortdc">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; -def int_ppc_tabortdci : GCCBuiltin<"__builtin_tabortdci">, +def int_ppc_tabortdci : ClangBuiltin<"__builtin_tabortdci">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; -def int_ppc_tcheck : GCCBuiltin<"__builtin_tcheck">, +def int_ppc_tcheck : ClangBuiltin<"__builtin_tcheck">, Intrinsic<[llvm_i32_ty], [], []>; -def int_ppc_treclaim : GCCBuiltin<"__builtin_treclaim">, +def int_ppc_treclaim : ClangBuiltin<"__builtin_treclaim">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; -def int_ppc_trechkpt : GCCBuiltin<"__builtin_trechkpt">, +def int_ppc_trechkpt : ClangBuiltin<"__builtin_trechkpt">, Intrinsic<[llvm_i32_ty], [], []>; -def int_ppc_tsr : GCCBuiltin<"__builtin_tsr">, +def int_ppc_tsr : ClangBuiltin<"__builtin_tsr">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; -def int_ppc_get_texasr : GCCBuiltin<"__builtin_get_texasr">, +def int_ppc_get_texasr : ClangBuiltin<"__builtin_get_texasr">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_get_texasru : GCCBuiltin<"__builtin_get_texasru">, +def int_ppc_get_texasru : ClangBuiltin<"__builtin_get_texasru">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_get_tfhar : GCCBuiltin<"__builtin_get_tfhar">, +def int_ppc_get_tfhar : ClangBuiltin<"__builtin_get_tfhar">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_get_tfiar : GCCBuiltin<"__builtin_get_tfiar">, +def int_ppc_get_tfiar : ClangBuiltin<"__builtin_get_tfiar">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_set_texasr : GCCBuiltin<"__builtin_set_texasr">, +def int_ppc_set_texasr : ClangBuiltin<"__builtin_set_texasr">, Intrinsic<[], [llvm_i64_ty], []>; -def int_ppc_set_texasru : GCCBuiltin<"__builtin_set_texasru">, +def int_ppc_set_texasru : ClangBuiltin<"__builtin_set_texasru">, Intrinsic<[], [llvm_i64_ty], []>; -def int_ppc_set_tfhar : GCCBuiltin<"__builtin_set_tfhar">, +def int_ppc_set_tfhar : ClangBuiltin<"__builtin_set_tfhar">, Intrinsic<[], [llvm_i64_ty], []>; -def int_ppc_set_tfiar : GCCBuiltin<"__builtin_set_tfiar">, +def int_ppc_set_tfiar : ClangBuiltin<"__builtin_set_tfiar">, Intrinsic<[], [llvm_i64_ty], []>; // Extended mnemonics -def int_ppc_tendall : GCCBuiltin<"__builtin_tendall">, +def int_ppc_tendall : ClangBuiltin<"__builtin_tendall">, Intrinsic<[llvm_i32_ty], [], []>; -def int_ppc_tresume : GCCBuiltin<"__builtin_tresume">, +def int_ppc_tresume : ClangBuiltin<"__builtin_tresume">, Intrinsic<[llvm_i32_ty], [], []>; -def int_ppc_tsuspend : GCCBuiltin<"__builtin_tsuspend">, +def int_ppc_tsuspend : ClangBuiltin<"__builtin_tsuspend">, Intrinsic<[llvm_i32_ty], [], []>; -def int_ppc_ttest : GCCBuiltin<"__builtin_ttest">, +def int_ppc_ttest : ClangBuiltin<"__builtin_ttest">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>; +// We currently use llvm.ppc.cfence in the context of atomic load which +// in LLVM IR requires its type to be one of integer, pointer and +// float point type. So llvm_any_ty here refers to type mentioned above. +// Backend is supposed to lower these types to appropriate MVTs. +def int_ppc_cfence : Intrinsic<[], [llvm_any_ty], []>; // PowerPC set FPSCR Intrinsic Definitions. -def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">, +def int_ppc_setrnd : ClangBuiltin<"__builtin_setrnd">, Intrinsic<[llvm_double_ty], [llvm_i32_ty], []>; } @@ -1552,218 +1590,212 @@ let TargetPrefix = "ppc" in { // XL Compat intrinsics. let TargetPrefix = "ppc" in { - def int_ppc_dcbfl : GCCBuiltin<"__builtin_ppc_dcbfl">, + def int_ppc_dcbfl : ClangBuiltin<"__builtin_ppc_dcbfl">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>; - def int_ppc_dcbflp : GCCBuiltin<"__builtin_ppc_dcbflp">, + def int_ppc_dcbflp : ClangBuiltin<"__builtin_ppc_dcbflp">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>; - def int_ppc_dcbst : GCCBuiltin<"__builtin_ppc_dcbst">, + def int_ppc_dcbst : ClangBuiltin<"__builtin_ppc_dcbst">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_ppc_dcbt : GCCBuiltin<"__builtin_ppc_dcbt">, + def int_ppc_dcbt : ClangBuiltin<"__builtin_ppc_dcbt">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; - def int_ppc_dcbtst : GCCBuiltin<"__builtin_ppc_dcbtst">, + def int_ppc_dcbtst : ClangBuiltin<"__builtin_ppc_dcbtst">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; - def int_ppc_dcbz : GCCBuiltin<"__builtin_ppc_dcbz">, + def int_ppc_dcbz : ClangBuiltin<"__builtin_ppc_dcbz">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_ppc_icbt : GCCBuiltin<"__builtin_ppc_icbt">, + def int_ppc_icbt : ClangBuiltin<"__builtin_ppc_icbt">, Intrinsic<[], [llvm_ptr_ty], []>; // Population Count in each Byte. def int_ppc_popcntb : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>; // sync instruction (i.e. sync 0, a.k.a hwsync) - def int_ppc_sync : GCCBuiltin<"__builtin_ppc_sync">, + def int_ppc_sync : ClangBuiltin<"__builtin_ppc_sync">, Intrinsic<[], [], []>; - def int_ppc_iospace_sync : GCCBuiltin<"__builtin_ppc_iospace_sync">, + def int_ppc_iospace_sync : ClangBuiltin<"__builtin_ppc_iospace_sync">, Intrinsic<[], [], []>; // isync instruction - def int_ppc_isync : GCCBuiltin<"__builtin_ppc_isync">, + def int_ppc_isync : ClangBuiltin<"__builtin_ppc_isync">, Intrinsic<[], [], []>; // lwsync is sync 1 - def int_ppc_lwsync : GCCBuiltin<"__builtin_ppc_lwsync">, + def int_ppc_lwsync : ClangBuiltin<"__builtin_ppc_lwsync">, Intrinsic<[], [], []>; - def int_ppc_iospace_lwsync : GCCBuiltin<"__builtin_ppc_iospace_lwsync">, + def int_ppc_iospace_lwsync : ClangBuiltin<"__builtin_ppc_iospace_lwsync">, Intrinsic<[], [], []>; // eieio instruction - def int_ppc_eieio : GCCBuiltin<"__builtin_ppc_eieio">, + def int_ppc_eieio : ClangBuiltin<"__builtin_ppc_eieio">, Intrinsic<[],[],[]>; - def int_ppc_iospace_eieio : GCCBuiltin<"__builtin_ppc_iospace_eieio">, + def int_ppc_iospace_eieio : ClangBuiltin<"__builtin_ppc_iospace_eieio">, Intrinsic<[],[],[]>; - def int_ppc_stdcx : GCCBuiltin<"__builtin_ppc_stdcx">, + def int_ppc_stdcx : ClangBuiltin<"__builtin_ppc_stdcx">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrWriteMem]>; - def int_ppc_stwcx : GCCBuiltin<"__builtin_ppc_stwcx">, + def int_ppc_stwcx : ClangBuiltin<"__builtin_ppc_stwcx">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrWriteMem]>; def int_ppc_sthcx : Intrinsic<[llvm_i32_ty], [ llvm_ptr_ty, llvm_i32_ty ], [IntrWriteMem]>; - def int_ppc_stbcx : GCCBuiltin<"__builtin_ppc_stbcx">, + def int_ppc_stbcx : ClangBuiltin<"__builtin_ppc_stbcx">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrWriteMem]>; - def int_ppc_dcbtstt : GCCBuiltin<"__builtin_ppc_dcbtstt">, + def int_ppc_dcbtstt : ClangBuiltin<"__builtin_ppc_dcbtstt">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; - def int_ppc_dcbtt : GCCBuiltin<"__builtin_ppc_dcbtt">, + def int_ppc_dcbtt : ClangBuiltin<"__builtin_ppc_dcbtt">, Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; - def int_ppc_mftbu : GCCBuiltin<"__builtin_ppc_mftbu">, + def int_ppc_mftbu : ClangBuiltin<"__builtin_ppc_mftbu">, Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; - def int_ppc_mfmsr : GCCBuiltin<"__builtin_ppc_mfmsr">, + def int_ppc_mfmsr : ClangBuiltin<"__builtin_ppc_mfmsr">, Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; def int_ppc_mfspr : Intrinsic<[llvm_anyint_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; def int_ppc_mtmsr - : GCCBuiltin<"__builtin_ppc_mtmsr">, Intrinsic<[], [llvm_i32_ty], []>; + : ClangBuiltin<"__builtin_ppc_mtmsr">, Intrinsic<[], [llvm_i32_ty], []>; def int_ppc_mtspr : Intrinsic<[], [llvm_i32_ty, llvm_anyint_ty], [ImmArg<ArgIndex<0>>]>; - def int_ppc_stfiw : GCCBuiltin<"__builtin_ppc_stfiw">, + def int_ppc_stfiw : ClangBuiltin<"__builtin_ppc_stfiw">, Intrinsic<[], [llvm_ptr_ty, llvm_double_ty], [IntrWriteMem]>; // compare def int_ppc_cmpeqb - : GCCBuiltin<"__builtin_ppc_cmpeqb">, + : ClangBuiltin<"__builtin_ppc_cmpeqb">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_cmprb - : GCCBuiltin<"__builtin_ppc_cmprb">, + : ClangBuiltin<"__builtin_ppc_cmprb">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<0>>]>; def int_ppc_setb - : GCCBuiltin<"__builtin_ppc_setb">, + : ClangBuiltin<"__builtin_ppc_setb">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_cmpb : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty, llvm_anyint_ty], [IntrNoMem]>; // multiply def int_ppc_mulhd - : GCCBuiltin<"__builtin_ppc_mulhd">, + : ClangBuiltin<"__builtin_ppc_mulhd">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_mulhdu - : GCCBuiltin<"__builtin_ppc_mulhdu">, + : ClangBuiltin<"__builtin_ppc_mulhdu">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_mulhw - : GCCBuiltin<"__builtin_ppc_mulhw">, + : ClangBuiltin<"__builtin_ppc_mulhw">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_ppc_mulhwu - : GCCBuiltin<"__builtin_ppc_mulhwu">, + : ClangBuiltin<"__builtin_ppc_mulhwu">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_ppc_maddhd - : GCCBuiltin<"__builtin_ppc_maddhd">, + : ClangBuiltin<"__builtin_ppc_maddhd">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_maddhdu - : GCCBuiltin<"__builtin_ppc_maddhdu">, + : ClangBuiltin<"__builtin_ppc_maddhdu">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_maddld - : GCCBuiltin<"__builtin_ppc_maddld">, + : ClangBuiltin<"__builtin_ppc_maddld">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; // load def int_ppc_load2r : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_load4r - : GCCBuiltin<"__builtin_ppc_load4r">, + : ClangBuiltin<"__builtin_ppc_load4r">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_load8r - : GCCBuiltin<"__builtin_ppc_load8r">, + : ClangBuiltin<"__builtin_ppc_load8r">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; // store def int_ppc_store2r - : GCCBuiltin<"__builtin_ppc_store2r">, + : ClangBuiltin<"__builtin_ppc_store2r">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], [IntrWriteMem]>; def int_ppc_store4r - : GCCBuiltin<"__builtin_ppc_store4r">, + : ClangBuiltin<"__builtin_ppc_store4r">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], [IntrWriteMem]>; def int_ppc_store8r - : GCCBuiltin<"__builtin_ppc_store8r">, + : ClangBuiltin<"__builtin_ppc_store8r">, Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], [IntrWriteMem]>; def int_ppc_insert_exp - : GCCBuiltin<"__builtin_ppc_insert_exp">, + : ClangBuiltin<"__builtin_ppc_insert_exp">, Intrinsic <[llvm_double_ty], [llvm_double_ty, llvm_i64_ty], [IntrNoMem]>; def int_ppc_extract_exp - : GCCBuiltin<"__builtin_ppc_extract_exp">, + : ClangBuiltin<"__builtin_ppc_extract_exp">, Intrinsic <[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_extract_sig - : GCCBuiltin<"__builtin_ppc_extract_sig">, + : ClangBuiltin<"__builtin_ppc_extract_sig">, Intrinsic <[llvm_i64_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_mtfsb0 - : GCCBuiltin<"__builtin_ppc_mtfsb0">, + : ClangBuiltin<"__builtin_ppc_mtfsb0">, Intrinsic <[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; def int_ppc_mtfsb1 - : GCCBuiltin<"__builtin_ppc_mtfsb1">, + : ClangBuiltin<"__builtin_ppc_mtfsb1">, Intrinsic <[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; def int_ppc_mtfsf : Intrinsic <[], [llvm_i32_ty, llvm_double_ty], [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; def int_ppc_mtfsfi - : GCCBuiltin<"__builtin_ppc_mtfsfi">, + : ClangBuiltin<"__builtin_ppc_mtfsfi">, Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>,ImmArg<ArgIndex<1>>]>; def int_ppc_fmsub - : GCCBuiltin<"__builtin_ppc_fmsub">, + : ClangBuiltin<"__builtin_ppc_fmsub">, Intrinsic <[llvm_double_ty], [llvm_double_ty, llvm_double_ty, llvm_double_ty], [IntrNoMem]>; def int_ppc_fmsubs - : GCCBuiltin<"__builtin_ppc_fmsubs">, + : ClangBuiltin<"__builtin_ppc_fmsubs">, Intrinsic <[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem]>; def int_ppc_fnmadd - : GCCBuiltin<"__builtin_ppc_fnmadd">, + : ClangBuiltin<"__builtin_ppc_fnmadd">, Intrinsic <[llvm_double_ty], [llvm_double_ty, llvm_double_ty, llvm_double_ty], [IntrNoMem]>; def int_ppc_fnmadds - : GCCBuiltin<"__builtin_ppc_fnmadds">, + : ClangBuiltin<"__builtin_ppc_fnmadds">, Intrinsic <[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem]>; def int_ppc_fnmsub - : GCCBuiltin<"__builtin_ppc_fnmsub">, - Intrinsic <[llvm_double_ty], - [llvm_double_ty, llvm_double_ty, llvm_double_ty], - [IntrNoMem]>; - def int_ppc_fnmsubs - : GCCBuiltin<"__builtin_ppc_fnmsubs">, - Intrinsic <[llvm_float_ty], - [llvm_float_ty, llvm_float_ty, llvm_float_ty], - [IntrNoMem]>; + : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem]>; def int_ppc_fre - : GCCBuiltin<"__builtin_ppc_fre">, + : ClangBuiltin<"__builtin_ppc_fre">, Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; def int_ppc_fres - : GCCBuiltin<"__builtin_ppc_fres">, + : ClangBuiltin<"__builtin_ppc_fres">, Intrinsic <[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; def int_ppc_addex - : GCCBuiltin<"__builtin_ppc_addex">, + : ClangBuiltin<"__builtin_ppc_addex">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<2>>]>; - def int_ppc_fsel : GCCBuiltin<"__builtin_ppc_fsel">, + def int_ppc_fsel : ClangBuiltin<"__builtin_ppc_fsel">, Intrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_ppc_fsels : GCCBuiltin<"__builtin_ppc_fsels">, + def int_ppc_fsels : ClangBuiltin<"__builtin_ppc_fsels">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem]>; - def int_ppc_frsqrte : GCCBuiltin<"__builtin_ppc_frsqrte">, + def int_ppc_frsqrte : ClangBuiltin<"__builtin_ppc_frsqrte">, Intrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; - def int_ppc_frsqrtes : GCCBuiltin<"__builtin_ppc_frsqrtes">, + def int_ppc_frsqrtes : ClangBuiltin<"__builtin_ppc_frsqrtes">, Intrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; - def int_ppc_compare_exp_uo : GCCBuiltin<"__builtin_ppc_compare_exp_uo">, + def int_ppc_compare_exp_uo : ClangBuiltin<"__builtin_ppc_compare_exp_uo">, Intrinsic<[llvm_i32_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_ppc_compare_exp_lt : GCCBuiltin<"__builtin_ppc_compare_exp_lt">, + def int_ppc_compare_exp_lt : ClangBuiltin<"__builtin_ppc_compare_exp_lt">, Intrinsic<[llvm_i32_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_ppc_compare_exp_gt : GCCBuiltin<"__builtin_ppc_compare_exp_gt">, + def int_ppc_compare_exp_gt : ClangBuiltin<"__builtin_ppc_compare_exp_gt">, Intrinsic<[llvm_i32_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; - def int_ppc_compare_exp_eq : GCCBuiltin<"__builtin_ppc_compare_exp_eq">, + def int_ppc_compare_exp_eq : ClangBuiltin<"__builtin_ppc_compare_exp_eq">, Intrinsic<[llvm_i32_ty], [llvm_double_ty, llvm_double_ty], [IntrNoMem]>; @@ -1773,6 +1805,12 @@ let TargetPrefix = "ppc" in { def int_ppc_test_data_class_f : Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; + def int_ppc_fnabs + : ClangBuiltin<"__builtin_ppc_fnabs">, + Intrinsic <[llvm_double_ty], [llvm_double_ty], [IntrNoMem]>; + def int_ppc_fnabss + : ClangBuiltin<"__builtin_ppc_fnabss">, + Intrinsic <[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>; def int_ppc_convert_f128_to_ppcf128 : Intrinsic<[llvm_ppcf128_ty], [llvm_f128_ty], [IntrNoMem]>; diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 6780436bd701..098ca1bc6cfb 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -140,7 +140,7 @@ let TargetPrefix = "riscv" in { // Vectors // The intrinsic does not have any operand that must be extended. -defvar NoSplatOperand = 0xF; +defvar NoScalarOperand = 0xF; // The intrinsic does not have a VL operand. // (e.g., riscv_vmv_x_s and riscv_vfmv_f_s) @@ -150,7 +150,7 @@ class RISCVVIntrinsic { // These intrinsics may accept illegal integer values in their llvm_any_ty // operand, so they have to be extended. Intrinsic IntrinsicID = !cast<Intrinsic>(NAME); - bits<4> SplatOperand = NoSplatOperand; + bits<4> ScalarOperand = NoScalarOperand; bits<5> VLOperand = NoVLOperand; } @@ -219,8 +219,8 @@ let TargetPrefix = "riscv" in { let VLOperand = 2; } // For unit stride load with mask - // Input: (maskedoff, pointer, mask, vl, ta) - class RISCVUSLoadMask + // Input: (maskedoff, pointer, mask, vl, policy) + class RISCVUSLoadMasked : Intrinsic<[llvm_anyvector_ty ], [LLVMMatchType<0>, LLVMPointerType<LLVMMatchType<0>>, @@ -231,11 +231,11 @@ let TargetPrefix = "riscv" in { let VLOperand = 3; } // For unit stride fault-only-first load with mask - // Input: (maskedoff, pointer, mask, vl, ta) + // Input: (maskedoff, pointer, mask, vl, policy) // Output: (data, vl) // NOTE: We model this with default memory properties since we model writing // VL as a side effect. IntrReadMem, IntrHasSideEffects does not work. - class RISCVUSLoadFFMask + class RISCVUSLoadFFMasked : Intrinsic<[llvm_anyvector_ty, llvm_anyint_ty], [LLVMMatchType<0>, LLVMPointerType<LLVMMatchType<0>>, @@ -255,8 +255,8 @@ let TargetPrefix = "riscv" in { let VLOperand = 3; } // For strided load with mask - // Input: (maskedoff, pointer, stride, mask, vl, ta) - class RISCVSLoadMask + // Input: (maskedoff, pointer, stride, mask, vl, policy) + class RISCVSLoadMasked : Intrinsic<[llvm_anyvector_ty ], [LLVMMatchType<0>, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, @@ -277,8 +277,8 @@ let TargetPrefix = "riscv" in { let VLOperand = 3; } // For indexed load with mask - // Input: (maskedoff, pointer, index, mask, vl, ta) - class RISCVILoadMask + // Input: (maskedoff, pointer, index, mask, vl, policy) + class RISCVILoadMasked : Intrinsic<[llvm_anyvector_ty ], [LLVMMatchType<0>, LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, @@ -300,7 +300,7 @@ let TargetPrefix = "riscv" in { } // For unit stride store with mask // Input: (vector_in, pointer, mask, vl) - class RISCVUSStoreMask + class RISCVUSStoreMasked : Intrinsic<[], [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, @@ -321,7 +321,7 @@ let TargetPrefix = "riscv" in { } // For stride store with mask // Input: (vector_in, pointer, stirde, mask, vl) - class RISCVSStoreMask + class RISCVSStoreMasked : Intrinsic<[], [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, @@ -341,7 +341,7 @@ let TargetPrefix = "riscv" in { } // For indexed store with mask // Input: (vector_in, pointer, index, mask, vl) - class RISCVIStoreMask + class RISCVIStoreMasked : Intrinsic<[], [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, @@ -350,16 +350,16 @@ let TargetPrefix = "riscv" in { let VLOperand = 4; } // For destination vector type is the same as source vector. - // Input: (vector_in, vl) - class RISCVUnaryAANoMask + // Input: (passthru, vector_in, vl) + class RISCVUnaryAAUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } // For destination vector type is the same as first source vector (with mask). - // Input: (vector_in, mask, vl, ta) - class RISCVUnaryAAMask + // Input: (vector_in, vector_in, mask, vl, policy) + class RISCVUnaryAAMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -367,7 +367,8 @@ let TargetPrefix = "riscv" in { [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 3; } - class RISCVUnaryAAMaskNoTA + // Input: (passthru, vector_in, vector_in, mask, vl) + class RISCVCompress : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], @@ -376,23 +377,24 @@ let TargetPrefix = "riscv" in { } // For destination vector type is the same as first and second source vector. // Input: (vector_in, vector_in, vl) - class RISCVBinaryAAANoMask + class RISCVBinaryAAAUnMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 2; } // For destination vector type is the same as first and second source vector. - // Input: (vector_in, int_vector_in, vl) - class RISCVRGatherVVNoMask + // Input: (passthru, vector_in, int_vector_in, vl) + class RISCVRGatherVVUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, LLVMVectorOfBitcastsToInt<0>, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, + LLVMVectorOfBitcastsToInt<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For destination vector type is the same as first and second source vector. - // Input: (vector_in, vector_in, int_vector_in, vl, ta) - class RISCVRGatherVVMask + // Input: (vector_in, vector_in, int_vector_in, vl, policy) + class RISCVRGatherVVMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMVectorOfBitcastsToInt<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -400,17 +402,18 @@ let TargetPrefix = "riscv" in { [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 4; } - // Input: (vector_in, int16_vector_in, vl) - class RISCVRGatherEI16VVNoMask + // Input: (passthru, vector_in, int16_vector_in, vl) + class RISCVRGatherEI16VVUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, + [LLVMMatchType<0>, LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For destination vector type is the same as first and second source vector. - // Input: (vector_in, vector_in, int16_vector_in, vl, ta) - class RISCVRGatherEI16VVMask + // Input: (vector_in, vector_in, int16_vector_in, vl, policy) + class RISCVRGatherEI16VVMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, @@ -421,17 +424,18 @@ let TargetPrefix = "riscv" in { } // For destination vector type is the same as first source vector, and the // second operand is XLen. - // Input: (vector_in, xlen_in, vl) - class RISCVGatherVXNoMask + // Input: (passthru, vector_in, xlen_in, vl) + class RISCVGatherVXUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_anyint_ty, LLVMMatchType<1>], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, + LLVMMatchType<1>], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For destination vector type is the same as first source vector (with mask). // Second operand is XLen. - // Input: (maskedoff, vector_in, xlen_in, mask, vl, ta) - class RISCVGatherVXMask + // Input: (maskedoff, vector_in, xlen_in, mask, vl, policy) + class RISCVGatherVXMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, @@ -440,38 +444,40 @@ let TargetPrefix = "riscv" in { let VLOperand = 4; } // For destination vector type is the same as first source vector. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVBinaryAAXNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVBinaryAAXUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; - let VLOperand = 2; + let ScalarOperand = 2; + let VLOperand = 3; } // For destination vector type is the same as first source vector (with mask). - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVBinaryAAXMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVBinaryAAXMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let ScalarOperand = 2; let VLOperand = 4; } // For destination vector type is the same as first source vector. The // second source operand must match the destination type or be an XLen scalar. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVBinaryAAShiftNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVBinaryAAShiftUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For destination vector type is the same as first source vector (with mask). // The second source operand must match the destination type or be an XLen scalar. - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVBinaryAAShiftMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVBinaryAAShiftMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -480,38 +486,40 @@ let TargetPrefix = "riscv" in { let VLOperand = 4; } // For destination vector type is NOT the same as first source vector. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVBinaryABXNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVBinaryABXUnMasked : Intrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; - let VLOperand = 2; + let ScalarOperand = 2; + let VLOperand = 3; } // For destination vector type is NOT the same as first source vector (with mask). - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVBinaryABXMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVBinaryABXMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<3>], [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let ScalarOperand = 2; let VLOperand = 4; } // For destination vector type is NOT the same as first source vector. The // second source operand must match the destination type or be an XLen scalar. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVBinaryABShiftNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVBinaryABShiftUnMasked : Intrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For destination vector type is NOT the same as first source vector (with mask). // The second source operand must match the destination type or be an XLen scalar. - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVBinaryABShiftMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVBinaryABShiftMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -520,15 +528,15 @@ let TargetPrefix = "riscv" in { let VLOperand = 4; } // For binary operations with V0 as input. - // Input: (vector_in, vector_in/scalar_in, V0, vl) + // Input: (passthru, vector_in, vector_in/scalar_in, V0, vl) class RISCVBinaryWithV0 : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_any_ty, + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; - let VLOperand = 3; + let ScalarOperand = 2; + let VLOperand = 4; } // For binary operations with mask type output and V0 as input. // Output: (mask type output) @@ -539,7 +547,7 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + let ScalarOperand = 1; let VLOperand = 3; } // For binary operations with mask type output. @@ -549,87 +557,91 @@ let TargetPrefix = "riscv" in { : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + let ScalarOperand = 1; let VLOperand = 2; } // For binary operations with mask type output without mask. // Output: (mask type output) // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVCompareNoMask + class RISCVCompareUnMasked : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + let ScalarOperand = 1; let VLOperand = 2; } // For binary operations with mask type output with mask. // Output: (mask type output) // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl) - class RISCVCompareMask + class RISCVCompareMasked : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let ScalarOperand = 2; let VLOperand = 4; } // For FP classify operations. // Output: (bit mask type output) - // Input: (vector_in, vl) - class RISCVClassifyNoMask + // Input: (passthru, vector_in, vl) + class RISCVClassifyUnMasked : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], - [llvm_anyvector_ty, llvm_anyint_ty], + [LLVMVectorOfBitcastsToInt<0>, llvm_anyvector_ty, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 1; } // For FP classify operations with mask. // Output: (bit mask type output) - // Input: (maskedoff, vector_in, mask, vl) - class RISCVClassifyMask + // Input: (maskedoff, vector_in, mask, vl, policy) + class RISCVClassifyMasked : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], [LLVMVectorOfBitcastsToInt<0>, llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyint_ty, LLVMMatchType<1>], + [IntrNoMem, ImmArg<ArgIndex<4>>]>, RISCVVIntrinsic { let VLOperand = 3; } // For Saturating binary operations. // The destination vector type is the same as first source vector. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVSaturatingBinaryAAXNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVSaturatingBinaryAAXUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let SplatOperand = 1; - let VLOperand = 2; + let ScalarOperand = 2; + let VLOperand = 3; } // For Saturating binary operations with mask. // The destination vector type is the same as first source vector. - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVSaturatingBinaryAAXMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVSaturatingBinaryAAXMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let SplatOperand = 2; + let ScalarOperand = 2; let VLOperand = 4; } // For Saturating binary operations. // The destination vector type is the same as first source vector. // The second source operand matches the destination type or is an XLen scalar. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVSaturatingBinaryAAShiftNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVSaturatingBinaryAAShiftUnMasked : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For Saturating binary operations with mask. // The destination vector type is the same as first source vector. // The second source operand matches the destination type or is an XLen scalar. - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVSaturatingBinaryAAShiftMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVSaturatingBinaryAAShiftMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -640,18 +652,19 @@ let TargetPrefix = "riscv" in { // For Saturating binary operations. // The destination vector type is NOT the same as first source vector. // The second source operand matches the destination type or is an XLen scalar. - // Input: (vector_in, vector_in/scalar_in, vl) - class RISCVSaturatingBinaryABShiftNoMask + // Input: (passthru, vector_in, vector_in/scalar_in, vl) + class RISCVSaturatingBinaryABShiftUnMasked : Intrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, + llvm_anyint_ty], [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let VLOperand = 2; + let VLOperand = 3; } // For Saturating binary operations with mask. // The destination vector type is NOT the same as first source vector (with mask). // The second source operand matches the destination type or is an XLen scalar. - // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) - class RISCVSaturatingBinaryABShiftMask + // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, policy) + class RISCVSaturatingBinaryABShiftMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -659,56 +672,69 @@ let TargetPrefix = "riscv" in { [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { let VLOperand = 4; } - class RISCVTernaryAAAXNoMask + // Input: (vector_in, vector_in, scalar_in, vl, policy) + class RVVSlideUnMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, - LLVMMatchType<1>], - [IntrNoMem]>, RISCVVIntrinsic { + LLVMMatchType<1>, LLVMMatchType<1>], + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 3; } - class RISCVTernaryAAAXMask + // Input: (vector_in, vector_in, vector_in/scalar_in, mask, vl, policy) + class RVVSlideMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>], - [IntrNoMem]>, RISCVVIntrinsic { + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<1>, LLVMMatchType<1>], + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 4; } - class RISCVTernaryAAXANoMask + // UnMasked Vector Multiply-Add operations, its first operand can not be undef. + // Input: (vector_in, vector_in/scalar, vector_in, vl, policy) + class RISCVTernaryAAXAUnMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, LLVMMatchType<0>, - llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + llvm_anyint_ty, LLVMMatchType<2>], + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { + let ScalarOperand = 1; let VLOperand = 3; } - class RISCVTernaryAAXAMask + // Masked Vector Multiply-Add operations, its first operand can not be undef. + // Input: (vector_in, vector_in/scalar, vector_in, mask, vl, policy + class RISCVTernaryAAXAMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyint_ty, LLVMMatchType<2>], + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let ScalarOperand = 1; let VLOperand = 4; } - class RISCVTernaryWideNoMask + // UnMasked Widening Vector Multiply-Add operations, its first operand can not be undef. + // Input: (vector_in, vector_in/scalar, vector_in, vl, policy) + class RISCVTernaryWideUnMasked : Intrinsic< [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyvector_ty, - llvm_anyint_ty], - [IntrNoMem] >, RISCVVIntrinsic { - let SplatOperand = 1; + llvm_anyint_ty, LLVMMatchType<3>], + [ImmArg<ArgIndex<4>>, IntrNoMem] >, RISCVVIntrinsic { + let ScalarOperand = 1; let VLOperand = 3; } - class RISCVTernaryWideMask + // Masked Widening Vector Multiply-Add operations, its first operand can not be undef. + // Input: (vector_in, vector_in/scalar, vector_in, mask, vl, policy + class RISCVTernaryWideMasked : Intrinsic< [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 1; + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyint_ty, LLVMMatchType<3>], + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let ScalarOperand = 1; let VLOperand = 4; } // For Reduction ternary operations. // For destination vector type is the same as first and third source vector. // Input: (vector_in, vector_in, vector_in, vl) - class RISCVReductionNoMask + class RISCVReductionUnMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>, llvm_anyint_ty], @@ -719,7 +745,7 @@ let TargetPrefix = "riscv" in { // For destination vector type is the same as first and third source vector. // The mask type come from second source vector. // Input: (maskedoff, vector_in, vector_in, vector_in, mask, vl) - class RISCVReductionMask + class RISCVReductionMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, llvm_anyint_ty], @@ -729,7 +755,7 @@ let TargetPrefix = "riscv" in { // For unary operations with scalar type output without mask // Output: (scalar type) // Input: (vector_in, vl) - class RISCVMaskUnarySOutNoMask + class RISCVMaskedUnarySOutUnMasked : Intrinsic<[LLVMMatchType<1>], [llvm_anyvector_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { @@ -738,23 +764,23 @@ let TargetPrefix = "riscv" in { // For unary operations with scalar type output with mask // Output: (scalar type) // Input: (vector_in, mask, vl) - class RISCVMaskUnarySOutMask + class RISCVMaskedUnarySOutMasked : Intrinsic<[LLVMMatchType<1>], [llvm_anyvector_ty, LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 2; } // For destination vector type is NOT the same as source vector. - // Input: (vector_in, vl) - class RISCVUnaryABNoMask + // Input: (passthru, vector_in, vl) + class RISCVUnaryABUnMasked : Intrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } // For destination vector type is NOT the same as source vector (with mask). - // Input: (maskedoff, vector_in, mask, vl, ta) - class RISCVUnaryABMask + // Input: (maskedoff, vector_in, mask, vl, policy) + class RISCVUnaryABMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, @@ -765,7 +791,7 @@ let TargetPrefix = "riscv" in { // For unary operations with the same vector type in/out without mask // Output: (vector) // Input: (vector_in, vl) - class RISCVUnaryNoMask + class RISCVUnaryUnMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { @@ -774,7 +800,7 @@ let TargetPrefix = "riscv" in { // For mask unary operations with mask type in/out with mask // Output: (mask type output) // Input: (mask type maskedoff, mask type vector_in, mask, vl) - class RISCVMaskUnaryMOutMask + class RISCVMaskedUnaryMOutMasked : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty], @@ -785,21 +811,28 @@ let TargetPrefix = "riscv" in { // Input: (vl) class RISCVNullaryIntrinsic : Intrinsic<[llvm_anyvector_ty], - [llvm_anyint_ty], + [llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } + // Output: (vector) + // Input: (passthru, vl) + class RISCVID + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 0; + let VLOperand = 1; } // For Conversion unary operations. - // Input: (vector_in, vl) - class RISCVConversionNoMask + // Input: (passthru, vector_in, vl) + class RISCVConversionUnMasked : Intrinsic<[llvm_anyvector_ty], - [llvm_anyvector_ty, llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } // For Conversion unary operations with mask. - // Input: (maskedoff, vector_in, mask, vl, ta) - class RISCVConversionMask + // Input: (maskedoff, vector_in, mask, vl, policy) + class RISCVConversionMasked : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, @@ -809,17 +842,18 @@ let TargetPrefix = "riscv" in { } // For unit stride segment load - // Input: (pointer, vl) + // Input: (passthru, pointer, vl) class RISCVUSSegLoad<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), - [LLVMPointerToElt<0>, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { - let VLOperand = 1; + !listconcat(!listsplat(LLVMMatchType<0>, nf), + [LLVMPointerToElt<0>, llvm_anyint_ty]), + [NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 1); } // For unit stride segment load with mask - // Input: (maskedoff, pointer, mask, vl, ta) - class RISCVUSSegLoadMask<int nf> + // Input: (maskedoff, pointer, mask, vl, policy) + class RISCVUSSegLoadMasked<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), !listconcat(!listsplat(LLVMMatchType<0>, nf), @@ -832,23 +866,24 @@ let TargetPrefix = "riscv" in { } // For unit stride fault-only-first segment load - // Input: (pointer, vl) + // Input: (passthru, pointer, vl) // Output: (data, vl) // NOTE: We model this with default memory properties since we model writing // VL as a side effect. IntrReadMem, IntrHasSideEffects does not work. class RISCVUSSegLoadFF<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), [llvm_anyint_ty]), - [LLVMPointerToElt<0>, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic { - let VLOperand = 1; + !listconcat(!listsplat(LLVMMatchType<0>, nf), + [LLVMPointerToElt<0>, LLVMMatchType<1>]), + [NoCapture<ArgIndex<nf>>]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 1); } // For unit stride fault-only-first segment load with mask - // Input: (maskedoff, pointer, mask, vl, ta) + // Input: (maskedoff, pointer, mask, vl, policy) // Output: (data, vl) // NOTE: We model this with default memory properties since we model writing // VL as a side effect. IntrReadMem, IntrHasSideEffects does not work. - class RISCVUSSegLoadFFMask<int nf> + class RISCVUSSegLoadFFMasked<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), [llvm_anyint_ty]), !listconcat(!listsplat(LLVMMatchType<0>, nf), @@ -861,17 +896,18 @@ let TargetPrefix = "riscv" in { } // For stride segment load - // Input: (pointer, offset, vl) + // Input: (passthru, pointer, offset, vl) class RISCVSSegLoad<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), - [LLVMPointerToElt<0>, llvm_anyint_ty, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { - let VLOperand = 2; + !listconcat(!listsplat(LLVMMatchType<0>, nf), + [LLVMPointerToElt<0>, llvm_anyint_ty, LLVMMatchType<1>]), + [NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 2); } // For stride segment load with mask - // Input: (maskedoff, pointer, offset, mask, vl, ta) - class RISCVSSegLoadMask<int nf> + // Input: (maskedoff, pointer, offset, mask, vl, policy) + class RISCVSSegLoadMasked<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), !listconcat(!listsplat(LLVMMatchType<0>, nf), @@ -885,17 +921,18 @@ let TargetPrefix = "riscv" in { } // For indexed segment load - // Input: (pointer, index, vl) + // Input: (passthru, pointer, index, vl) class RISCVISegLoad<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), - [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { - let VLOperand = 2; + !listconcat(!listsplat(LLVMMatchType<0>, nf), + [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty]), + [NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 2); } // For indexed segment load with mask - // Input: (maskedoff, pointer, index, mask, vl, ta) - class RISCVISegLoadMask<int nf> + // Input: (maskedoff, pointer, index, mask, vl, policy) + class RISCVISegLoadMasked<int nf> : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), !listconcat(!listsplat(LLVMMatchType<0>, nf), @@ -920,7 +957,7 @@ let TargetPrefix = "riscv" in { } // For unit stride segment store with mask // Input: (value, pointer, mask, vl) - class RISCVUSSegStoreMask<int nf> + class RISCVUSSegStoreMasked<int nf> : Intrinsic<[], !listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), @@ -944,7 +981,7 @@ let TargetPrefix = "riscv" in { } // For stride segment store with mask // Input: (value, pointer, offset, mask, vl) - class RISCVSSegStoreMask<int nf> + class RISCVSSegStoreMasked<int nf> : Intrinsic<[], !listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), @@ -968,7 +1005,7 @@ let TargetPrefix = "riscv" in { } // For indexed segment store with mask // Input: (value, pointer, offset, mask, vl) - class RISCVISegStoreMask<int nf> + class RISCVISegStoreMasked<int nf> : Intrinsic<[], !listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), @@ -981,76 +1018,76 @@ let TargetPrefix = "riscv" in { multiclass RISCVUSLoad { def "int_riscv_" # NAME : RISCVUSLoad; - def "int_riscv_" # NAME # "_mask" : RISCVUSLoadMask; + def "int_riscv_" # NAME # "_mask" : RISCVUSLoadMasked; } multiclass RISCVUSLoadFF { def "int_riscv_" # NAME : RISCVUSLoadFF; - def "int_riscv_" # NAME # "_mask" : RISCVUSLoadFFMask; + def "int_riscv_" # NAME # "_mask" : RISCVUSLoadFFMasked; } multiclass RISCVSLoad { def "int_riscv_" # NAME : RISCVSLoad; - def "int_riscv_" # NAME # "_mask" : RISCVSLoadMask; + def "int_riscv_" # NAME # "_mask" : RISCVSLoadMasked; } multiclass RISCVILoad { def "int_riscv_" # NAME : RISCVILoad; - def "int_riscv_" # NAME # "_mask" : RISCVILoadMask; + def "int_riscv_" # NAME # "_mask" : RISCVILoadMasked; } multiclass RISCVUSStore { def "int_riscv_" # NAME : RISCVUSStore; - def "int_riscv_" # NAME # "_mask" : RISCVUSStoreMask; + def "int_riscv_" # NAME # "_mask" : RISCVUSStoreMasked; } multiclass RISCVSStore { def "int_riscv_" # NAME : RISCVSStore; - def "int_riscv_" # NAME # "_mask" : RISCVSStoreMask; + def "int_riscv_" # NAME # "_mask" : RISCVSStoreMasked; } multiclass RISCVIStore { def "int_riscv_" # NAME : RISCVIStore; - def "int_riscv_" # NAME # "_mask" : RISCVIStoreMask; + def "int_riscv_" # NAME # "_mask" : RISCVIStoreMasked; } multiclass RISCVUnaryAA { - def "int_riscv_" # NAME : RISCVUnaryAANoMask; - def "int_riscv_" # NAME # "_mask" : RISCVUnaryAAMask; + def "int_riscv_" # NAME : RISCVUnaryAAUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVUnaryAAMasked; } multiclass RISCVUnaryAB { - def "int_riscv_" # NAME : RISCVUnaryABNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVUnaryABMask; + def "int_riscv_" # NAME : RISCVUnaryABUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVUnaryABMasked; } // AAX means the destination type(A) is the same as the first source // type(A). X means any type for the second source operand. multiclass RISCVBinaryAAX { - def "int_riscv_" # NAME : RISCVBinaryAAXNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVBinaryAAXMask; + def "int_riscv_" # NAME : RISCVBinaryAAXUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVBinaryAAXMasked; } // Like RISCVBinaryAAX, but the second operand is used a shift amount so it // must be a vector or an XLen scalar. multiclass RISCVBinaryAAShift { - def "int_riscv_" # NAME : RISCVBinaryAAShiftNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVBinaryAAShiftMask; + def "int_riscv_" # NAME : RISCVBinaryAAShiftUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVBinaryAAShiftMasked; } multiclass RISCVRGatherVV { - def "int_riscv_" # NAME : RISCVRGatherVVNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVRGatherVVMask; + def "int_riscv_" # NAME : RISCVRGatherVVUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVRGatherVVMasked; } multiclass RISCVRGatherVX { - def "int_riscv_" # NAME : RISCVGatherVXNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVGatherVXMask; + def "int_riscv_" # NAME : RISCVGatherVXUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVGatherVXMasked; } multiclass RISCVRGatherEI16VV { - def "int_riscv_" # NAME : RISCVRGatherEI16VVNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVRGatherEI16VVMask; + def "int_riscv_" # NAME : RISCVRGatherEI16VVUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVRGatherEI16VVMasked; } // ABX means the destination type(A) is different from the first source // type(B). X means any type for the second source operand. multiclass RISCVBinaryABX { - def "int_riscv_" # NAME : RISCVBinaryABXNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVBinaryABXMask; + def "int_riscv_" # NAME : RISCVBinaryABXUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVBinaryABXMasked; } // Like RISCVBinaryABX, but the second operand is used a shift amount so it // must be a vector or an XLen scalar. multiclass RISCVBinaryABShift { - def "int_riscv_" # NAME : RISCVBinaryABShiftNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVBinaryABShiftMask; + def "int_riscv_" # NAME : RISCVBinaryABShiftUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVBinaryABShiftMasked; } multiclass RISCVBinaryWithV0 { def "int_riscv_" # NAME : RISCVBinaryWithV0; @@ -1062,80 +1099,80 @@ let TargetPrefix = "riscv" in { def "int_riscv_" # NAME : RISCVBinaryMOut; } multiclass RISCVSaturatingBinaryAAX { - def "int_riscv_" # NAME : RISCVSaturatingBinaryAAXNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryAAXMask; + def "int_riscv_" # NAME : RISCVSaturatingBinaryAAXUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryAAXMasked; } multiclass RISCVSaturatingBinaryAAShift { - def "int_riscv_" # NAME : RISCVSaturatingBinaryAAShiftNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryAAShiftMask; + def "int_riscv_" # NAME : RISCVSaturatingBinaryAAShiftUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryAAShiftMasked; } multiclass RISCVSaturatingBinaryABShift { - def "int_riscv_" # NAME : RISCVSaturatingBinaryABShiftNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryABShiftMask; + def "int_riscv_" # NAME : RISCVSaturatingBinaryABShiftUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVSaturatingBinaryABShiftMasked; } - multiclass RISCVTernaryAAAX { - def "int_riscv_" # NAME : RISCVTernaryAAAXNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVTernaryAAAXMask; + multiclass RVVSlide { + def "int_riscv_" # NAME : RVVSlideUnMasked; + def "int_riscv_" # NAME # "_mask" : RVVSlideMasked; } multiclass RISCVTernaryAAXA { - def "int_riscv_" # NAME : RISCVTernaryAAXANoMask; - def "int_riscv_" # NAME # "_mask" : RISCVTernaryAAXAMask; + def "int_riscv_" # NAME : RISCVTernaryAAXAUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVTernaryAAXAMasked; } multiclass RISCVCompare { - def "int_riscv_" # NAME : RISCVCompareNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVCompareMask; + def "int_riscv_" # NAME : RISCVCompareUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVCompareMasked; } multiclass RISCVClassify { - def "int_riscv_" # NAME : RISCVClassifyNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVClassifyMask; + def "int_riscv_" # NAME : RISCVClassifyUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVClassifyMasked; } multiclass RISCVTernaryWide { - def "int_riscv_" # NAME : RISCVTernaryWideNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVTernaryWideMask; + def "int_riscv_" # NAME : RISCVTernaryWideUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVTernaryWideMasked; } multiclass RISCVReduction { - def "int_riscv_" # NAME : RISCVReductionNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVReductionMask; + def "int_riscv_" # NAME : RISCVReductionUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVReductionMasked; } - multiclass RISCVMaskUnarySOut { - def "int_riscv_" # NAME : RISCVMaskUnarySOutNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVMaskUnarySOutMask; + multiclass RISCVMaskedUnarySOut { + def "int_riscv_" # NAME : RISCVMaskedUnarySOutUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVMaskedUnarySOutMasked; } - multiclass RISCVMaskUnaryMOut { - def "int_riscv_" # NAME : RISCVUnaryNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVMaskUnaryMOutMask; + multiclass RISCVMaskedUnaryMOut { + def "int_riscv_" # NAME : RISCVUnaryUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVMaskedUnaryMOutMasked; } multiclass RISCVConversion { - def "int_riscv_" #NAME :RISCVConversionNoMask; - def "int_riscv_" # NAME # "_mask" : RISCVConversionMask; + def "int_riscv_" #NAME :RISCVConversionUnMasked; + def "int_riscv_" # NAME # "_mask" : RISCVConversionMasked; } multiclass RISCVUSSegLoad<int nf> { def "int_riscv_" # NAME : RISCVUSSegLoad<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadMasked<nf>; } multiclass RISCVUSSegLoadFF<int nf> { def "int_riscv_" # NAME : RISCVUSSegLoadFF<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadFFMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadFFMasked<nf>; } multiclass RISCVSSegLoad<int nf> { def "int_riscv_" # NAME : RISCVSSegLoad<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVSSegLoadMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVSSegLoadMasked<nf>; } multiclass RISCVISegLoad<int nf> { def "int_riscv_" # NAME : RISCVISegLoad<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVISegLoadMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVISegLoadMasked<nf>; } multiclass RISCVUSSegStore<int nf> { def "int_riscv_" # NAME : RISCVUSSegStore<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVUSSegStoreMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVUSSegStoreMasked<nf>; } multiclass RISCVSSegStore<int nf> { def "int_riscv_" # NAME : RISCVSSegStore<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVSSegStoreMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVSSegStoreMasked<nf>; } multiclass RISCVISegStore<int nf> { def "int_riscv_" # NAME : RISCVISegStore<nf>; - def "int_riscv_" # NAME # "_mask" : RISCVISegStoreMask<nf>; + def "int_riscv_" # NAME # "_mask" : RISCVISegStoreMasked<nf>; } defm vle : RISCVUSLoad; @@ -1242,20 +1279,29 @@ let TargetPrefix = "riscv" in { defm vmerge : RISCVBinaryWithV0; + // Output: (vector) + // Input: (passthru, vector_in, vl) def int_riscv_vmv_v_v : Intrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } + // Output: (vector) + // Input: (passthru, scalar, vl) def int_riscv_vmv_v_x : Intrinsic<[llvm_anyint_ty], - [LLVMVectorElementType<0>, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMVectorElementType<0>, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } + // Output: (vector) + // Input: (passthru, scalar, vl) def int_riscv_vfmv_v_f : Intrinsic<[llvm_anyfloat_ty], - [LLVMVectorElementType<0>, llvm_anyint_ty], + [LLVMMatchType<0>, LLVMVectorElementType<0>, + llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } def int_riscv_vmv_x_s : Intrinsic<[LLVMVectorElementType<0>], @@ -1313,8 +1359,8 @@ let TargetPrefix = "riscv" in { defm vfmerge : RISCVBinaryWithV0; - defm vslideup : RISCVTernaryAAAX; - defm vslidedown : RISCVTernaryAAAX; + defm vslideup : RVVSlide; + defm vslidedown : RVVSlide; defm vslide1up : RISCVBinaryAAX; defm vslide1down : RISCVBinaryAAX; @@ -1325,7 +1371,7 @@ let TargetPrefix = "riscv" in { defm vrgather_vx : RISCVRGatherVX; defm vrgatherei16_vv : RISCVRGatherEI16VV; - def "int_riscv_vcompress" : RISCVUnaryAAMaskNoTA; + def "int_riscv_vcompress" : RISCVCompress; defm vaaddu : RISCVSaturatingBinaryAAX; defm vaadd : RISCVSaturatingBinaryAAX; @@ -1367,22 +1413,22 @@ let TargetPrefix = "riscv" in { defm vfwredusum : RISCVReduction; defm vfwredosum : RISCVReduction; - def int_riscv_vmand: RISCVBinaryAAANoMask; - def int_riscv_vmnand: RISCVBinaryAAANoMask; - def int_riscv_vmandn: RISCVBinaryAAANoMask; - def int_riscv_vmxor: RISCVBinaryAAANoMask; - def int_riscv_vmor: RISCVBinaryAAANoMask; - def int_riscv_vmnor: RISCVBinaryAAANoMask; - def int_riscv_vmorn: RISCVBinaryAAANoMask; - def int_riscv_vmxnor: RISCVBinaryAAANoMask; + def int_riscv_vmand: RISCVBinaryAAAUnMasked; + def int_riscv_vmnand: RISCVBinaryAAAUnMasked; + def int_riscv_vmandn: RISCVBinaryAAAUnMasked; + def int_riscv_vmxor: RISCVBinaryAAAUnMasked; + def int_riscv_vmor: RISCVBinaryAAAUnMasked; + def int_riscv_vmnor: RISCVBinaryAAAUnMasked; + def int_riscv_vmorn: RISCVBinaryAAAUnMasked; + def int_riscv_vmxnor: RISCVBinaryAAAUnMasked; def int_riscv_vmclr : RISCVNullaryIntrinsic; def int_riscv_vmset : RISCVNullaryIntrinsic; - defm vcpop : RISCVMaskUnarySOut; - defm vfirst : RISCVMaskUnarySOut; - defm vmsbf : RISCVMaskUnaryMOut; - defm vmsof : RISCVMaskUnaryMOut; - defm vmsif : RISCVMaskUnaryMOut; + defm vcpop : RISCVMaskedUnarySOut; + defm vfirst : RISCVMaskedUnarySOut; + defm vmsbf : RISCVMaskedUnaryMOut; + defm vmsof : RISCVMaskedUnaryMOut; + defm vmsif : RISCVMaskedUnaryMOut; defm vfcvt_xu_f_v : RISCVConversion; defm vfcvt_x_f_v : RISCVConversion; @@ -1409,34 +1455,35 @@ let TargetPrefix = "riscv" in { defm vfncvt_rod_f_f_w : RISCVConversion; // Output: (vector) - // Input: (mask type input, vl) + // Input: (passthru, mask type input, vl) def int_riscv_viota : Intrinsic<[llvm_anyvector_ty], - [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let VLOperand = 1; + let VLOperand = 2; } // Output: (vector) - // Input: (maskedoff, mask type vector_in, mask, vl) + // Input: (maskedoff, mask type vector_in, mask, vl, policy) def int_riscv_viota_mask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { + llvm_anyint_ty, LLVMMatchType<1>], + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 3; } // Output: (vector) - // Input: (vl) - def int_riscv_vid : RISCVNullaryIntrinsic; + // Input: (passthru, vl) + def int_riscv_vid : RISCVID; // Output: (vector) - // Input: (maskedoff, mask, vl) + // Input: (maskedoff, mask, vl, policy) def int_riscv_vid_mask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, - llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic { + llvm_anyint_ty, LLVMMatchType<1>], + [ImmArg<ArgIndex<3>>, IntrNoMem]>, RISCVVIntrinsic { let VLOperand = 2; } @@ -1463,6 +1510,16 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, llvm_anyptr_ty, llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [NoCapture<ArgIndex<1>>, IntrWriteMem]>; + + // Segment loads for fixed vectors. + foreach nf = [2, 3, 4, 5, 6, 7, 8] in { + def int_riscv_seg # nf # _load + : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, + !add(nf, -1))), + [llvm_anyptr_ty, llvm_anyint_ty], + [NoCapture<ArgIndex<0>>, IntrReadMem]>; + } + } // TargetPrefix = "riscv" //===----------------------------------------------------------------------===// @@ -1503,7 +1560,7 @@ class ScalarCryptoByteSelectAny : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i8_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, - ImmArg<ArgIndex<2>>, Returned<ArgIndex<0>>]>; + ImmArg<ArgIndex<2>>]>; // Zknd def int_riscv_aes32dsi : ScalarCryptoByteSelect32; diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td new file mode 100644 index 000000000000..14c628595d30 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -0,0 +1,31 @@ +//===- IntrinsicsSPIRV.td - Defines SPIRV intrinsics -------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines all of the SPIRV-specific intrinsics. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "spv" in { + def int_spv_assign_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>; + def int_spv_assign_name : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>; + + def int_spv_track_constant : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty]>; + def int_spv_init_global : Intrinsic<[], [llvm_any_ty, llvm_any_ty]>; + def int_spv_unref_global : Intrinsic<[], [llvm_any_ty]>; + + def int_spv_gep : Intrinsic<[llvm_anyptr_ty], [llvm_i1_ty, llvm_any_ty, llvm_vararg_ty], [ImmArg<ArgIndex<0>>]>; + def int_spv_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i8_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; + def int_spv_store : Intrinsic<[], [llvm_i32_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i8_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>; + def int_spv_extractv : Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_vararg_ty]>; + def int_spv_insertv : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_any_ty, llvm_vararg_ty]>; + def int_spv_extractelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_anyint_ty]>; + def int_spv_insertelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_anyint_ty]>; + def int_spv_const_composite : Intrinsic<[llvm_i32_ty], [llvm_vararg_ty]>; + def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>; + def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>; +} diff --git a/llvm/include/llvm/IR/IntrinsicsSystemZ.td b/llvm/include/llvm/IR/IntrinsicsSystemZ.td index a149b571072c..d881a1126bf2 100644 --- a/llvm/include/llvm/IR/IntrinsicsSystemZ.td +++ b/llvm/include/llvm/IR/IntrinsicsSystemZ.td @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// class SystemZUnaryConv<string name, LLVMType result, LLVMType arg> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[result], [arg], [IntrNoMem]>; class SystemZUnary<string name, LLVMType type> @@ -24,14 +24,14 @@ class SystemZUnaryCC<LLVMType type> : SystemZUnaryConvCC<type, type>; class SystemZBinaryConv<string name, LLVMType result, LLVMType arg> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[result], [arg, arg], [IntrNoMem]>; class SystemZBinary<string name, LLVMType type> : SystemZBinaryConv<name, type, type>; class SystemZBinaryInt<string name, LLVMType type> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[type], [type, llvm_i32_ty], [IntrNoMem]>; class SystemZBinaryConvCC<LLVMType result, LLVMType arg> @@ -45,7 +45,7 @@ class SystemZBinaryCC<LLVMType type> : SystemZBinaryConvCC<type, type>; class SystemZTernaryConv<string name, LLVMType result, LLVMType arg> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[result], [arg, arg, result], [IntrNoMem]>; class SystemZTernaryConvCC<LLVMType result, LLVMType arg> @@ -55,7 +55,7 @@ class SystemZTernary<string name, LLVMType type> : SystemZTernaryConv<name, type, type>; class SystemZTernaryInt<string name, LLVMType type> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[type], [type, type, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; class SystemZTernaryIntCC<LLVMType type> @@ -63,7 +63,7 @@ class SystemZTernaryIntCC<LLVMType type> [IntrNoMem, ImmArg<ArgIndex<2>>]>; class SystemZQuaternaryInt<string name, LLVMType type> - : GCCBuiltin<"__builtin_s390_" # name>, + : ClangBuiltin<"__builtin_s390_" # name>, Intrinsic<[type], [type, type, type, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; @@ -216,16 +216,16 @@ let TargetPrefix = "s390" in { def int_s390_tabort : Intrinsic<[], [llvm_i64_ty], [IntrNoReturn, Throws, IntrWriteMem]>; - def int_s390_tend : GCCBuiltin<"__builtin_tend">, + def int_s390_tend : ClangBuiltin<"__builtin_tend">, Intrinsic<[llvm_i32_ty], []>; - def int_s390_etnd : GCCBuiltin<"__builtin_tx_nesting_depth">, + def int_s390_etnd : ClangBuiltin<"__builtin_tx_nesting_depth">, Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; def int_s390_ntstg : Intrinsic<[], [llvm_i64_ty, llvm_ptr64_ty], [IntrArgMemOnly, IntrWriteMem]>; - def int_s390_ppa_txassist : GCCBuiltin<"__builtin_tx_assist">, + def int_s390_ppa_txassist : ClangBuiltin<"__builtin_tx_assist">, Intrinsic<[], [llvm_i32_ty]>; } @@ -236,24 +236,24 @@ let TargetPrefix = "s390" in { //===----------------------------------------------------------------------===// let TargetPrefix = "s390" in { - def int_s390_lcbb : GCCBuiltin<"__builtin_s390_lcbb">, + def int_s390_lcbb : ClangBuiltin<"__builtin_s390_lcbb">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_s390_vlbb : GCCBuiltin<"__builtin_s390_vlbb">, + def int_s390_vlbb : ClangBuiltin<"__builtin_s390_vlbb">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>; - def int_s390_vll : GCCBuiltin<"__builtin_s390_vll">, + def int_s390_vll : ClangBuiltin<"__builtin_s390_vll">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_s390_vpdi : GCCBuiltin<"__builtin_s390_vpdi">, + def int_s390_vpdi : ClangBuiltin<"__builtin_s390_vpdi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_s390_vperm : GCCBuiltin<"__builtin_s390_vperm">, + def int_s390_vperm : ClangBuiltin<"__builtin_s390_vperm">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -264,7 +264,7 @@ let TargetPrefix = "s390" in { defm int_s390_vpkls : SystemZBinaryTruncHFG<"vpkls">; defm int_s390_vpkls : SystemZBinaryTruncCCHFG; - def int_s390_vstl : GCCBuiltin<"__builtin_s390_vstl">, + def int_s390_vstl : ClangBuiltin<"__builtin_s390_vstl">, Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], [IntrArgMemOnly, IntrWriteMem]>; @@ -314,7 +314,7 @@ let TargetPrefix = "s390" in { def int_s390_vsrl : SystemZBinary<"vsrl", llvm_v16i8_ty>; def int_s390_vsrlb : SystemZBinary<"vsrlb", llvm_v16i8_ty>; - def int_s390_vsldb : GCCBuiltin<"__builtin_s390_vsldb">, + def int_s390_vsldb : ClangBuiltin<"__builtin_s390_vsldb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; @@ -382,7 +382,7 @@ let TargetPrefix = "s390" in { def int_s390_vbperm : SystemZBinaryConv<"vbperm", llvm_v2i64_ty, llvm_v16i8_ty>; - def int_s390_vmslg : GCCBuiltin<"__builtin_s390_vmslg">, + def int_s390_vmslg : ClangBuiltin<"__builtin_s390_vmslg">, Intrinsic<[llvm_v16i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; @@ -411,21 +411,21 @@ let TargetPrefix = "s390" in { [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; // Instructions from the Vector Packed Decimal Facility - def int_s390_vlrl : GCCBuiltin<"__builtin_s390_vlrl">, + def int_s390_vlrl : ClangBuiltin<"__builtin_s390_vlrl">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_s390_vstrl : GCCBuiltin<"__builtin_s390_vstrl">, + def int_s390_vstrl : ClangBuiltin<"__builtin_s390_vstrl">, Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], [IntrArgMemOnly, IntrWriteMem]>; // Instructions from the Vector Enhancements Facility 2 - def int_s390_vsld : GCCBuiltin<"__builtin_s390_vsld">, + def int_s390_vsld : ClangBuiltin<"__builtin_s390_vsld">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_s390_vsrd : GCCBuiltin<"__builtin_s390_vsrd">, + def int_s390_vsrd : ClangBuiltin<"__builtin_s390_vsrd">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; @@ -438,23 +438,23 @@ let TargetPrefix = "s390" in { def int_s390_vstrszf : SystemZTernaryConvCC<llvm_v16i8_ty, llvm_v4i32_ty>; // Instructions from the NNP-assist Facility - def int_s390_vclfnhs : GCCBuiltin<"__builtin_s390_vclfnhs">, + def int_s390_vclfnhs : ClangBuiltin<"__builtin_s390_vclfnhs">, Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_s390_vclfnls : GCCBuiltin<"__builtin_s390_vclfnls">, + def int_s390_vclfnls : ClangBuiltin<"__builtin_s390_vclfnls">, Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_s390_vcrnfs : GCCBuiltin<"__builtin_s390_vcrnfs">, + def int_s390_vcrnfs : ClangBuiltin<"__builtin_s390_vcrnfs">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_s390_vcfn : GCCBuiltin<"__builtin_s390_vcfn">, + def int_s390_vcfn : ClangBuiltin<"__builtin_s390_vcfn">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_s390_vcnf : GCCBuiltin<"__builtin_s390_vcnf">, + def int_s390_vcnf : ClangBuiltin<"__builtin_s390_vcnf">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; @@ -467,9 +467,9 @@ let TargetPrefix = "s390" in { //===----------------------------------------------------------------------===// let TargetPrefix = "s390" in { - def int_s390_sfpc : GCCBuiltin<"__builtin_s390_sfpc">, + def int_s390_sfpc : ClangBuiltin<"__builtin_s390_sfpc">, Intrinsic<[], [llvm_i32_ty], []>; - def int_s390_efpc : GCCBuiltin<"__builtin_s390_efpc">, + def int_s390_efpc : ClangBuiltin<"__builtin_s390_efpc">, Intrinsic<[llvm_i32_ty], [], []>; def int_s390_tdc : Intrinsic<[llvm_i32_ty], [llvm_anyfloat_ty, llvm_i64_ty], diff --git a/llvm/include/llvm/IR/IntrinsicsVE.td b/llvm/include/llvm/IR/IntrinsicsVE.td index be4bccef0cc1..15b828b320ea 100644 --- a/llvm/include/llvm/IR/IntrinsicsVE.td +++ b/llvm/include/llvm/IR/IntrinsicsVE.td @@ -2,31 +2,28 @@ // VEL Intrinsic instructions. let TargetPrefix = "ve" in { - def int_ve_vl_svob : GCCBuiltin<"__builtin_ve_vl_svob">, - Intrinsic<[], [], [IntrHasSideEffects]>; - - def int_ve_vl_pack_f32p : GCCBuiltin<"__builtin_ve_vl_pack_f32p">, + def int_ve_vl_pack_f32p : ClangBuiltin<"__builtin_ve_vl_pack_f32p">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], [IntrReadMem]>; - def int_ve_vl_pack_f32a : GCCBuiltin<"__builtin_ve_vl_pack_f32a">, + def int_ve_vl_pack_f32a : ClangBuiltin<"__builtin_ve_vl_pack_f32a">, Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrReadMem]>; def int_ve_vl_extract_vm512u : - GCCBuiltin<"__builtin_ve_vl_extract_vm512u">, + ClangBuiltin<"__builtin_ve_vl_extract_vm512u">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v512i1>], [IntrNoMem]>; def int_ve_vl_extract_vm512l : - GCCBuiltin<"__builtin_ve_vl_extract_vm512l">, + ClangBuiltin<"__builtin_ve_vl_extract_vm512l">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v512i1>], [IntrNoMem]>; def int_ve_vl_insert_vm512u : - GCCBuiltin<"__builtin_ve_vl_insert_vm512u">, + ClangBuiltin<"__builtin_ve_vl_insert_vm512u">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v256i1>], [IntrNoMem]>; def int_ve_vl_insert_vm512l : - GCCBuiltin<"__builtin_ve_vl_insert_vm512l">, + ClangBuiltin<"__builtin_ve_vl_insert_vm512l">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v256i1>], [IntrNoMem]>; } diff --git a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td index 67cbd307903d..554dd8557200 100644 --- a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td +++ b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td @@ -1,1213 +1,1257 @@ -let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : GCCBuiltin<"__builtin_ve_vl_vld_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : GCCBuiltin<"__builtin_ve_vl_vld_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : GCCBuiltin<"__builtin_ve_vl_vldu_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : GCCBuiltin<"__builtin_ve_vl_vldu_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : GCCBuiltin<"__builtin_ve_vl_vldunc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldunc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : GCCBuiltin<"__builtin_ve_vl_vldlsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : GCCBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : GCCBuiltin<"__builtin_ve_vl_vldlzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : GCCBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : GCCBuiltin<"__builtin_ve_vl_vld2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : GCCBuiltin<"__builtin_ve_vl_vld2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : GCCBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : GCCBuiltin<"__builtin_ve_vl_vldu2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : GCCBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : GCCBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : GCCBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : GCCBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : GCCBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : GCCBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : GCCBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst_vssl : GCCBuiltin<"__builtin_ve_vl_vst_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst_vssml : GCCBuiltin<"__builtin_ve_vl_vst_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssl : GCCBuiltin<"__builtin_ve_vl_vstnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssml : GCCBuiltin<"__builtin_ve_vl_vstnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstot_vssl : GCCBuiltin<"__builtin_ve_vl_vstot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstot_vssml : GCCBuiltin<"__builtin_ve_vl_vstot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu_vssl : GCCBuiltin<"__builtin_ve_vl_vstu_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu_vssml : GCCBuiltin<"__builtin_ve_vl_vstu_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstunc_vssl : GCCBuiltin<"__builtin_ve_vl_vstunc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstunc_vssml : GCCBuiltin<"__builtin_ve_vl_vstunc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstuot_vssl : GCCBuiltin<"__builtin_ve_vl_vstuot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstuot_vssml : GCCBuiltin<"__builtin_ve_vl_vstuot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstuncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstuncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstuncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstuncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl_vssl : GCCBuiltin<"__builtin_ve_vl_vstl_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl_vssml : GCCBuiltin<"__builtin_ve_vl_vstl_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlnc_vssl : GCCBuiltin<"__builtin_ve_vl_vstlnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlnc_vssml : GCCBuiltin<"__builtin_ve_vl_vstlnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlot_vssl : GCCBuiltin<"__builtin_ve_vl_vstlot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlot_vssml : GCCBuiltin<"__builtin_ve_vl_vstlot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstlncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstlncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstlncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2d_vssl : GCCBuiltin<"__builtin_ve_vl_vst2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2d_vssml : GCCBuiltin<"__builtin_ve_vl_vst2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dnc_vssl : GCCBuiltin<"__builtin_ve_vl_vst2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dnc_vssml : GCCBuiltin<"__builtin_ve_vl_vst2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dot_vssl : GCCBuiltin<"__builtin_ve_vl_vst2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dot_vssml : GCCBuiltin<"__builtin_ve_vl_vst2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dncot_vssl : GCCBuiltin<"__builtin_ve_vl_vst2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vst2dncot_vssml : GCCBuiltin<"__builtin_ve_vl_vst2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2d_vssl : GCCBuiltin<"__builtin_ve_vl_vstu2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2d_vssml : GCCBuiltin<"__builtin_ve_vl_vstu2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dnc_vssl : GCCBuiltin<"__builtin_ve_vl_vstu2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dnc_vssml : GCCBuiltin<"__builtin_ve_vl_vstu2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dot_vssl : GCCBuiltin<"__builtin_ve_vl_vstu2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dot_vssml : GCCBuiltin<"__builtin_ve_vl_vstu2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstu2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstu2dncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstu2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2d_vssl : GCCBuiltin<"__builtin_ve_vl_vstl2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2d_vssml : GCCBuiltin<"__builtin_ve_vl_vstl2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dnc_vssl : GCCBuiltin<"__builtin_ve_vl_vstl2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dnc_vssml : GCCBuiltin<"__builtin_ve_vl_vstl2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dot_vssl : GCCBuiltin<"__builtin_ve_vl_vstl2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dot_vssml : GCCBuiltin<"__builtin_ve_vl_vstl2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstl2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstl2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pfchv_ssl : GCCBuiltin<"__builtin_ve_vl_pfchv_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>; -let TargetPrefix = "ve" in def int_ve_vl_pfchvnc_ssl : GCCBuiltin<"__builtin_ve_vl_pfchvnc_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>; -let TargetPrefix = "ve" in def int_ve_vl_lsv_vvss : GCCBuiltin<"__builtin_ve_vl_lsv_vvss">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i64>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lvsl_svs : GCCBuiltin<"__builtin_ve_vl_lvsl_svs">, Intrinsic<[LLVMType<i64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lvsd_svs : GCCBuiltin<"__builtin_ve_vl_lvsd_svs">, Intrinsic<[LLVMType<f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lvss_svs : GCCBuiltin<"__builtin_ve_vl_lvss_svs">, Intrinsic<[LLVMType<f32>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lvm_mmss : GCCBuiltin<"__builtin_ve_vl_lvm_mmss">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lvm_MMss : GCCBuiltin<"__builtin_ve_vl_lvm_MMss">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_svm_sms : GCCBuiltin<"__builtin_ve_vl_svm_sms">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i64>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_svm_sMs : GCCBuiltin<"__builtin_ve_vl_svm_sMs">, Intrinsic<[LLVMType<i64>], [LLVMType<v512i1>, LLVMType<i64>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsl : GCCBuiltin<"__builtin_ve_vl_vbrdd_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsvl : GCCBuiltin<"__builtin_ve_vl_vbrdd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsmvl : GCCBuiltin<"__builtin_ve_vl_vbrdd_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsl : GCCBuiltin<"__builtin_ve_vl_vbrdl_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsvl : GCCBuiltin<"__builtin_ve_vl_vbrdl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsmvl : GCCBuiltin<"__builtin_ve_vl_vbrdl_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsl : GCCBuiltin<"__builtin_ve_vl_vbrds_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsvl : GCCBuiltin<"__builtin_ve_vl_vbrds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsmvl : GCCBuiltin<"__builtin_ve_vl_vbrds_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsl : GCCBuiltin<"__builtin_ve_vl_vbrdw_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsvl : GCCBuiltin<"__builtin_ve_vl_vbrdw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsmvl : GCCBuiltin<"__builtin_ve_vl_vbrdw_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsl : GCCBuiltin<"__builtin_ve_vl_pvbrd_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsvl : GCCBuiltin<"__builtin_ve_vl_pvbrd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsMvl : GCCBuiltin<"__builtin_ve_vl_pvbrd_vsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vvvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vvvvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vsvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vsvvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vand_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vvvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vvvvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vsvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vsvvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vor_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvmvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvmvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vseq_vl : GCCBuiltin<"__builtin_ve_vl_vseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vseq_vvl : GCCBuiltin<"__builtin_ve_vl_vseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vl : GCCBuiltin<"__builtin_ve_vl_pvseqlo_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vvl : GCCBuiltin<"__builtin_ve_vl_pvseqlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vl : GCCBuiltin<"__builtin_ve_vl_pvsequp_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vvl : GCCBuiltin<"__builtin_ve_vl_pvsequp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvseq_vl : GCCBuiltin<"__builtin_ve_vl_pvseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvseq_vvl : GCCBuiltin<"__builtin_ve_vl_pvseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssvl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvl : GCCBuiltin<"__builtin_ve_vl_vfsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvl : GCCBuiltin<"__builtin_ve_vl_vfsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvl : GCCBuiltin<"__builtin_ve_vl_vrcpd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvvl : GCCBuiltin<"__builtin_ve_vl_vrcpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvl : GCCBuiltin<"__builtin_ve_vl_vrcps_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvvl : GCCBuiltin<"__builtin_ve_vl_vrcps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvl : GCCBuiltin<"__builtin_ve_vl_pvrcp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrcp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrt_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrt_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvMvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvMvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtdw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtdw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtdl_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtdl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtds_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtsd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtsd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvml : GCCBuiltin<"__builtin_ve_vl_vmrg_vvvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmrg_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvml : GCCBuiltin<"__builtin_ve_vl_vmrg_vsvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmrg_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vvvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMvl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vsvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMvl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsl : GCCBuiltin<"__builtin_ve_vl_vshf_vvvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsvl : GCCBuiltin<"__builtin_ve_vl_vshf_vvvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vcp_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcp_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vex_vvmvl : GCCBuiltin<"__builtin_ve_vl_vex_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklat_ml : GCCBuiltin<"__builtin_ve_vl_vfmklat_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklaf_ml : GCCBuiltin<"__builtin_ve_vl_vfmklaf_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkat_Ml : GCCBuiltin<"__builtin_ve_vl_pvfmkat_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkaf_Ml : GCCBuiltin<"__builtin_ve_vl_pvfmkaf_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkllt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkllt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkleq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkleq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkleqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkleqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkllenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkllenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkweq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkweq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkweqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkweqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkweq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkweq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkweqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkweqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkslt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkslt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkseq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkseq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkseqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkseqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkslenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkslenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkslt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkseq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkseq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkseqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkseqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkslenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvl : GCCBuiltin<"__builtin_ve_vl_vsumwsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvml : GCCBuiltin<"__builtin_ve_vl_vsumwsx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvl : GCCBuiltin<"__builtin_ve_vl_vsumwzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvml : GCCBuiltin<"__builtin_ve_vl_vsumwzx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvl : GCCBuiltin<"__builtin_ve_vl_vsuml_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvml : GCCBuiltin<"__builtin_ve_vl_vsuml_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvl : GCCBuiltin<"__builtin_ve_vl_vfsumd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvml : GCCBuiltin<"__builtin_ve_vl_vfsumd_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvl : GCCBuiltin<"__builtin_ve_vl_vfsums_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvml : GCCBuiltin<"__builtin_ve_vl_vfsums_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvl : GCCBuiltin<"__builtin_ve_vl_vrminslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvl : GCCBuiltin<"__builtin_ve_vl_vrminsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmindfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmindfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmindlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmindlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrminsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrminsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrminslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrminslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrand_vvl : GCCBuiltin<"__builtin_ve_vl_vrand_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrand_vvml : GCCBuiltin<"__builtin_ve_vl_vrand_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vror_vvl : GCCBuiltin<"__builtin_ve_vl_vror_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vror_vvml : GCCBuiltin<"__builtin_ve_vl_vror_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvl : GCCBuiltin<"__builtin_ve_vl_vrxor_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvml : GCCBuiltin<"__builtin_ve_vl_vrxor_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : GCCBuiltin<"__builtin_ve_vl_vgt_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssl : GCCBuiltin<"__builtin_ve_vl_vsc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssml : GCCBuiltin<"__builtin_ve_vl_vsc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vscnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vscnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssl : GCCBuiltin<"__builtin_ve_vl_vscu_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssml : GCCBuiltin<"__builtin_ve_vl_vscu_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssl : GCCBuiltin<"__builtin_ve_vl_vscunc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssml : GCCBuiltin<"__builtin_ve_vl_vscunc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscuot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscuot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscuncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscuncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssl : GCCBuiltin<"__builtin_ve_vl_vscl_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssml : GCCBuiltin<"__builtin_ve_vl_vscl_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; -let TargetPrefix = "ve" in def int_ve_vl_andm_mmm : GCCBuiltin<"__builtin_ve_vl_andm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_andm_MMM : GCCBuiltin<"__builtin_ve_vl_andm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_orm_mmm : GCCBuiltin<"__builtin_ve_vl_orm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_orm_MMM : GCCBuiltin<"__builtin_ve_vl_orm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_xorm_mmm : GCCBuiltin<"__builtin_ve_vl_xorm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_xorm_MMM : GCCBuiltin<"__builtin_ve_vl_xorm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_eqvm_mmm : GCCBuiltin<"__builtin_ve_vl_eqvm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_eqvm_MMM : GCCBuiltin<"__builtin_ve_vl_eqvm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_nndm_mmm : GCCBuiltin<"__builtin_ve_vl_nndm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_nndm_MMM : GCCBuiltin<"__builtin_ve_vl_nndm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_negm_mm : GCCBuiltin<"__builtin_ve_vl_negm_mm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_negm_MM : GCCBuiltin<"__builtin_ve_vl_negm_MM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_pcvm_sml : GCCBuiltin<"__builtin_ve_vl_pcvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_lzvm_sml : GCCBuiltin<"__builtin_ve_vl_lzvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; -let TargetPrefix = "ve" in def int_ve_vl_tovm_sml : GCCBuiltin<"__builtin_ve_vl_tovm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : ClangBuiltin<"__builtin_ve_vl_vld_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : ClangBuiltin<"__builtin_ve_vl_vld_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : ClangBuiltin<"__builtin_ve_vl_vldu_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst_vssl : ClangBuiltin<"__builtin_ve_vl_vst_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst_vssml : ClangBuiltin<"__builtin_ve_vl_vst_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssml : ClangBuiltin<"__builtin_ve_vl_vstnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstot_vssl : ClangBuiltin<"__builtin_ve_vl_vstot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstot_vssml : ClangBuiltin<"__builtin_ve_vl_vstot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstncot_vssl : ClangBuiltin<"__builtin_ve_vl_vstncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstncot_vssml : ClangBuiltin<"__builtin_ve_vl_vstncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu_vssl : ClangBuiltin<"__builtin_ve_vl_vstu_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu_vssml : ClangBuiltin<"__builtin_ve_vl_vstu_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstunc_vssl : ClangBuiltin<"__builtin_ve_vl_vstunc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstunc_vssml : ClangBuiltin<"__builtin_ve_vl_vstunc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstuot_vssl : ClangBuiltin<"__builtin_ve_vl_vstuot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstuot_vssml : ClangBuiltin<"__builtin_ve_vl_vstuot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstuncot_vssl : ClangBuiltin<"__builtin_ve_vl_vstuncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstuncot_vssml : ClangBuiltin<"__builtin_ve_vl_vstuncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl_vssl : ClangBuiltin<"__builtin_ve_vl_vstl_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl_vssml : ClangBuiltin<"__builtin_ve_vl_vstl_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstlnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlnc_vssml : ClangBuiltin<"__builtin_ve_vl_vstlnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlot_vssl : ClangBuiltin<"__builtin_ve_vl_vstlot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlot_vssml : ClangBuiltin<"__builtin_ve_vl_vstlot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlncot_vssl : ClangBuiltin<"__builtin_ve_vl_vstlncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstlncot_vssml : ClangBuiltin<"__builtin_ve_vl_vstlncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2d_vssl : ClangBuiltin<"__builtin_ve_vl_vst2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2d_vssml : ClangBuiltin<"__builtin_ve_vl_vst2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vst2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dnc_vssml : ClangBuiltin<"__builtin_ve_vl_vst2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dot_vssl : ClangBuiltin<"__builtin_ve_vl_vst2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dot_vssml : ClangBuiltin<"__builtin_ve_vl_vst2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dncot_vssl : ClangBuiltin<"__builtin_ve_vl_vst2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vst2dncot_vssml : ClangBuiltin<"__builtin_ve_vl_vst2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vstu2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2d_vssml : ClangBuiltin<"__builtin_ve_vl_vstu2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstu2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dnc_vssml : ClangBuiltin<"__builtin_ve_vl_vstu2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dot_vssl : ClangBuiltin<"__builtin_ve_vl_vstu2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dot_vssml : ClangBuiltin<"__builtin_ve_vl_vstu2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dncot_vssl : ClangBuiltin<"__builtin_ve_vl_vstu2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstu2dncot_vssml : ClangBuiltin<"__builtin_ve_vl_vstu2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2d_vssl : ClangBuiltin<"__builtin_ve_vl_vstl2d_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2d_vssml : ClangBuiltin<"__builtin_ve_vl_vstl2d_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstl2dnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dnc_vssml : ClangBuiltin<"__builtin_ve_vl_vstl2dnc_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dot_vssl : ClangBuiltin<"__builtin_ve_vl_vstl2dot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dot_vssml : ClangBuiltin<"__builtin_ve_vl_vstl2dot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssl : ClangBuiltin<"__builtin_ve_vl_vstl2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssml : ClangBuiltin<"__builtin_ve_vl_vstl2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pfchv_ssl : ClangBuiltin<"__builtin_ve_vl_pfchv_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>; +let TargetPrefix = "ve" in def int_ve_vl_pfchvnc_ssl : ClangBuiltin<"__builtin_ve_vl_pfchvnc_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>; +let TargetPrefix = "ve" in def int_ve_vl_lsv_vvss : ClangBuiltin<"__builtin_ve_vl_lsv_vvss">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lvsl_svs : ClangBuiltin<"__builtin_ve_vl_lvsl_svs">, Intrinsic<[LLVMType<i64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lvsd_svs : ClangBuiltin<"__builtin_ve_vl_lvsd_svs">, Intrinsic<[LLVMType<f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lvss_svs : ClangBuiltin<"__builtin_ve_vl_lvss_svs">, Intrinsic<[LLVMType<f32>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lvm_mmss : ClangBuiltin<"__builtin_ve_vl_lvm_mmss">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lvm_MMss : ClangBuiltin<"__builtin_ve_vl_lvm_MMss">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_svm_sms : ClangBuiltin<"__builtin_ve_vl_svm_sms">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_svm_sMs : ClangBuiltin<"__builtin_ve_vl_svm_sMs">, Intrinsic<[LLVMType<i64>], [LLVMType<v512i1>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsl : ClangBuiltin<"__builtin_ve_vl_vbrdd_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsvl : ClangBuiltin<"__builtin_ve_vl_vbrdd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdd_vsmvl : ClangBuiltin<"__builtin_ve_vl_vbrdd_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsl : ClangBuiltin<"__builtin_ve_vl_vbrdl_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsvl : ClangBuiltin<"__builtin_ve_vl_vbrdl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdl_vsmvl : ClangBuiltin<"__builtin_ve_vl_vbrdl_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsl : ClangBuiltin<"__builtin_ve_vl_vbrds_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsvl : ClangBuiltin<"__builtin_ve_vl_vbrds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrds_vsmvl : ClangBuiltin<"__builtin_ve_vl_vbrds_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsl : ClangBuiltin<"__builtin_ve_vl_vbrdw_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsvl : ClangBuiltin<"__builtin_ve_vl_vbrdw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrdw_vsmvl : ClangBuiltin<"__builtin_ve_vl_vbrdw_vsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsl : ClangBuiltin<"__builtin_ve_vl_pvbrd_vsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsvl : ClangBuiltin<"__builtin_ve_vl_pvbrd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsMvl : ClangBuiltin<"__builtin_ve_vl_pvbrd_vsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvl : ClangBuiltin<"__builtin_ve_vl_vmv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vaddul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vadduw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvaddu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vaddswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vaddswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvadds_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vaddsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vsubul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vsubuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvsubu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vsubswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vsubswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvsubs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vsubsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmulul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmuluw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmulswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmulswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmulsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvl : ClangBuiltin<"__builtin_ve_vl_vmulslw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmulslw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvl : ClangBuiltin<"__builtin_ve_vl_vmulslw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmulslw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vdivul_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vdivuw_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vdivswsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vdivswzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vdivsl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvcmpu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvcmps_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vcmpsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvmaxs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vminswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vminswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvmins_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmaxsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vminsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vvvl : ClangBuiltin<"__builtin_ve_vl_vand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vvvvl : ClangBuiltin<"__builtin_ve_vl_vand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vsvl : ClangBuiltin<"__builtin_ve_vl_vand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vsvvl : ClangBuiltin<"__builtin_ve_vl_vand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vand_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vand_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vand_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvl : ClangBuiltin<"__builtin_ve_vl_pvand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvl : ClangBuiltin<"__builtin_ve_vl_pvand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvand_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvand_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vvvl : ClangBuiltin<"__builtin_ve_vl_vor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vvvvl : ClangBuiltin<"__builtin_ve_vl_vor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vsvl : ClangBuiltin<"__builtin_ve_vl_vor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vsvvl : ClangBuiltin<"__builtin_ve_vl_vor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vor_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvl : ClangBuiltin<"__builtin_ve_vl_pvor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvl : ClangBuiltin<"__builtin_ve_vl_pvor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvl : ClangBuiltin<"__builtin_ve_vl_vxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvvl : ClangBuiltin<"__builtin_ve_vl_vxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvl : ClangBuiltin<"__builtin_ve_vl_vxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvvl : ClangBuiltin<"__builtin_ve_vl_vxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vxor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vxor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvxor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvl : ClangBuiltin<"__builtin_ve_vl_veqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvvl : ClangBuiltin<"__builtin_ve_vl_veqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvl : ClangBuiltin<"__builtin_ve_vl_veqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvvl : ClangBuiltin<"__builtin_ve_vl_veqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvmvl : ClangBuiltin<"__builtin_ve_vl_veqv_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvmvl : ClangBuiltin<"__builtin_ve_vl_veqv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pveqv_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldz_vvl : ClangBuiltin<"__builtin_ve_vl_vldz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldz_vvvl : ClangBuiltin<"__builtin_ve_vl_vldz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vldz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vldz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzlo_vvl : ClangBuiltin<"__builtin_ve_vl_pvldzlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzlo_vvvl : ClangBuiltin<"__builtin_ve_vl_pvldzlo_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzlo_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvldzlo_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzup_vvl : ClangBuiltin<"__builtin_ve_vl_pvldzup_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzup_vvvl : ClangBuiltin<"__builtin_ve_vl_pvldzup_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldzup_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvldzup_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldz_vvl : ClangBuiltin<"__builtin_ve_vl_pvldz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldz_vvvl : ClangBuiltin<"__builtin_ve_vl_pvldz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvldz_vvMvl : ClangBuiltin<"__builtin_ve_vl_pvldz_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vpcnt_vvl : ClangBuiltin<"__builtin_ve_vl_vpcnt_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vpcnt_vvvl : ClangBuiltin<"__builtin_ve_vl_vpcnt_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vpcnt_vvmvl : ClangBuiltin<"__builtin_ve_vl_vpcnt_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntlo_vvl : ClangBuiltin<"__builtin_ve_vl_pvpcntlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntlo_vvvl : ClangBuiltin<"__builtin_ve_vl_pvpcntlo_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntlo_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvpcntlo_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntup_vvl : ClangBuiltin<"__builtin_ve_vl_pvpcntup_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntup_vvvl : ClangBuiltin<"__builtin_ve_vl_pvpcntup_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcntup_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvpcntup_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcnt_vvl : ClangBuiltin<"__builtin_ve_vl_pvpcnt_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcnt_vvvl : ClangBuiltin<"__builtin_ve_vl_pvpcnt_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvpcnt_vvMvl : ClangBuiltin<"__builtin_ve_vl_pvpcnt_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrv_vvl : ClangBuiltin<"__builtin_ve_vl_vbrv_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrv_vvvl : ClangBuiltin<"__builtin_ve_vl_vbrv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vbrv_vvmvl : ClangBuiltin<"__builtin_ve_vl_vbrv_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvlo_vvl : ClangBuiltin<"__builtin_ve_vl_pvbrvlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvlo_vvvl : ClangBuiltin<"__builtin_ve_vl_pvbrvlo_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvlo_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvbrvlo_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvup_vvl : ClangBuiltin<"__builtin_ve_vl_pvbrvup_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvup_vvvl : ClangBuiltin<"__builtin_ve_vl_pvbrvup_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrvup_vvmvl : ClangBuiltin<"__builtin_ve_vl_pvbrvup_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrv_vvl : ClangBuiltin<"__builtin_ve_vl_pvbrv_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrv_vvvl : ClangBuiltin<"__builtin_ve_vl_pvbrv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvbrv_vvMvl : ClangBuiltin<"__builtin_ve_vl_pvbrv_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vseq_vl : ClangBuiltin<"__builtin_ve_vl_vseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vseq_vvl : ClangBuiltin<"__builtin_ve_vl_vseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vl : ClangBuiltin<"__builtin_ve_vl_pvseqlo_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vvl : ClangBuiltin<"__builtin_ve_vl_pvseqlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vl : ClangBuiltin<"__builtin_ve_vl_pvsequp_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vvl : ClangBuiltin<"__builtin_ve_vl_pvsequp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvseq_vl : ClangBuiltin<"__builtin_ve_vl_pvseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvseq_vvl : ClangBuiltin<"__builtin_ve_vl_pvseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvl : ClangBuiltin<"__builtin_ve_vl_vsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsl : ClangBuiltin<"__builtin_ve_vl_vsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsvl : ClangBuiltin<"__builtin_ve_vl_vsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsll_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vsll_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsMvl : ClangBuiltin<"__builtin_ve_vl_pvsll_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsvl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vsrl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsMvl : ClangBuiltin<"__builtin_ve_vl_pvsrl_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vslawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vslawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsMvl : ClangBuiltin<"__builtin_ve_vl_pvsla_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvl : ClangBuiltin<"__builtin_ve_vl_vslal_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvvl : ClangBuiltin<"__builtin_ve_vl_vslal_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsl : ClangBuiltin<"__builtin_ve_vl_vslal_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsvl : ClangBuiltin<"__builtin_ve_vl_vslal_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vslal_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vslal_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vsrawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsvl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vsrawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsMvl : ClangBuiltin<"__builtin_ve_vl_pvsra_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvl : ClangBuiltin<"__builtin_ve_vl_vsral_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvvl : ClangBuiltin<"__builtin_ve_vl_vsral_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsl : ClangBuiltin<"__builtin_ve_vl_vsral_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsvl : ClangBuiltin<"__builtin_ve_vl_vsral_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vsral_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsmvl : ClangBuiltin<"__builtin_ve_vl_vsral_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssl : ClangBuiltin<"__builtin_ve_vl_vsfa_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssvl : ClangBuiltin<"__builtin_ve_vl_vsfa_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vsfa_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfaddd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfadds_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfadd_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfsubd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfsubs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfsub_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmuld_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmuls_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmul_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfdivd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfdivs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvl : ClangBuiltin<"__builtin_ve_vl_vfsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvl : ClangBuiltin<"__builtin_ve_vl_vfsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvvl : ClangBuiltin<"__builtin_ve_vl_vfsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfcmpd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfcmps_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfcmp_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmaxd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmaxs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmax_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmind_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmins_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmin_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvmvl : ClangBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvMvl : ClangBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvl : ClangBuiltin<"__builtin_ve_vl_vrcpd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvvl : ClangBuiltin<"__builtin_ve_vl_vrcpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvl : ClangBuiltin<"__builtin_ve_vl_vrcps_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvvl : ClangBuiltin<"__builtin_ve_vl_vrcps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvl : ClangBuiltin<"__builtin_ve_vl_pvrcp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvvl : ClangBuiltin<"__builtin_ve_vl_pvrcp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvl : ClangBuiltin<"__builtin_ve_vl_vrsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvvl : ClangBuiltin<"__builtin_ve_vl_vrsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvl : ClangBuiltin<"__builtin_ve_vl_pvrsqrt_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvvl : ClangBuiltin<"__builtin_ve_vl_pvrsqrt_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvvl : ClangBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvl : ClangBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvvl : ClangBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvl : ClangBuiltin<"__builtin_ve_vl_pvcvtws_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvvl : ClangBuiltin<"__builtin_ve_vl_pvcvtws_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvMvl : ClangBuiltin<"__builtin_ve_vl_pvcvtws_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvl : ClangBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvvl : ClangBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvMvl : ClangBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtld_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtld_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtldrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtldrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcvtldrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtdw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtdw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvl : ClangBuiltin<"__builtin_ve_vl_pvcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvvl : ClangBuiltin<"__builtin_ve_vl_pvcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtdl_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtdl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtds_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvl : ClangBuiltin<"__builtin_ve_vl_vcvtsd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvvl : ClangBuiltin<"__builtin_ve_vl_vcvtsd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvml : ClangBuiltin<"__builtin_ve_vl_vmrg_vvvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvmvl : ClangBuiltin<"__builtin_ve_vl_vmrg_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvml : ClangBuiltin<"__builtin_ve_vl_vmrg_vsvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvmvl : ClangBuiltin<"__builtin_ve_vl_vmrg_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMl : ClangBuiltin<"__builtin_ve_vl_vmrgw_vvvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMvl : ClangBuiltin<"__builtin_ve_vl_vmrgw_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMl : ClangBuiltin<"__builtin_ve_vl_vmrgw_vsvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMvl : ClangBuiltin<"__builtin_ve_vl_vmrgw_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsl : ClangBuiltin<"__builtin_ve_vl_vshf_vvvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsvl : ClangBuiltin<"__builtin_ve_vl_vshf_vvvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vcp_vvmvl : ClangBuiltin<"__builtin_ve_vl_vcp_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vex_vvmvl : ClangBuiltin<"__builtin_ve_vl_vex_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklat_ml : ClangBuiltin<"__builtin_ve_vl_vfmklat_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklaf_ml : ClangBuiltin<"__builtin_ve_vl_vfmklaf_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkat_Ml : ClangBuiltin<"__builtin_ve_vl_pvfmkat_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkaf_Ml : ClangBuiltin<"__builtin_ve_vl_pvfmkaf_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkllt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkllt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkleq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkleq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkleqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkleqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmklgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmklgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkllenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkllenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkweq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkweq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkweqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkweqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkwlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkwlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkweq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkweq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkweqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkweqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkwlenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkdlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkdlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkslt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkslt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkseq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkseq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkseqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkseqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmksgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmksgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvl : ClangBuiltin<"__builtin_ve_vl_vfmkslenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvml : ClangBuiltin<"__builtin_ve_vl_vfmkslenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvml : ClangBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkslt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkseq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkseq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkseqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkseqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmksgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmksgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_Mvl : ClangBuiltin<"__builtin_ve_vl_pvfmkslenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_MvMl : ClangBuiltin<"__builtin_ve_vl_pvfmkslenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvl : ClangBuiltin<"__builtin_ve_vl_vsumwsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvml : ClangBuiltin<"__builtin_ve_vl_vsumwsx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvl : ClangBuiltin<"__builtin_ve_vl_vsumwzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvml : ClangBuiltin<"__builtin_ve_vl_vsumwzx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvl : ClangBuiltin<"__builtin_ve_vl_vsuml_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvml : ClangBuiltin<"__builtin_ve_vl_vsuml_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvl : ClangBuiltin<"__builtin_ve_vl_vfsumd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvml : ClangBuiltin<"__builtin_ve_vl_vfsumd_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvl : ClangBuiltin<"__builtin_ve_vl_vfsums_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvml : ClangBuiltin<"__builtin_ve_vl_vfsums_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvl : ClangBuiltin<"__builtin_ve_vl_vrminswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvl : ClangBuiltin<"__builtin_ve_vl_vrminswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvl : ClangBuiltin<"__builtin_ve_vl_vrminswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvl : ClangBuiltin<"__builtin_ve_vl_vrminswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvl : ClangBuiltin<"__builtin_ve_vl_vrmaxsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvvl : ClangBuiltin<"__builtin_ve_vl_vrmaxsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvl : ClangBuiltin<"__builtin_ve_vl_vrminslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvl : ClangBuiltin<"__builtin_ve_vl_vrminsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvvl : ClangBuiltin<"__builtin_ve_vl_vrminsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmaxslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmindfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmindfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrmindlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrmindlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrminsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrminsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvl : ClangBuiltin<"__builtin_ve_vl_vfrminslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvvl : ClangBuiltin<"__builtin_ve_vl_vfrminslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrand_vvl : ClangBuiltin<"__builtin_ve_vl_vrand_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrand_vvml : ClangBuiltin<"__builtin_ve_vl_vrand_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vror_vvl : ClangBuiltin<"__builtin_ve_vl_vror_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vror_vvml : ClangBuiltin<"__builtin_ve_vl_vror_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvl : ClangBuiltin<"__builtin_ve_vl_vrxor_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvml : ClangBuiltin<"__builtin_ve_vl_vrxor_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : ClangBuiltin<"__builtin_ve_vl_vgt_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssl : ClangBuiltin<"__builtin_ve_vl_vsc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssml : ClangBuiltin<"__builtin_ve_vl_vsc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vscnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vscnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssl : ClangBuiltin<"__builtin_ve_vl_vscot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssml : ClangBuiltin<"__builtin_ve_vl_vscot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssl : ClangBuiltin<"__builtin_ve_vl_vscncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssml : ClangBuiltin<"__builtin_ve_vl_vscncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssl : ClangBuiltin<"__builtin_ve_vl_vscu_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssml : ClangBuiltin<"__builtin_ve_vl_vscu_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vscunc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vscunc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssl : ClangBuiltin<"__builtin_ve_vl_vscuot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssml : ClangBuiltin<"__builtin_ve_vl_vscuot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssl : ClangBuiltin<"__builtin_ve_vl_vscuncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssml : ClangBuiltin<"__builtin_ve_vl_vscuncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssl : ClangBuiltin<"__builtin_ve_vl_vscl_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssml : ClangBuiltin<"__builtin_ve_vl_vscl_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vsclnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vsclnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssl : ClangBuiltin<"__builtin_ve_vl_vsclot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssml : ClangBuiltin<"__builtin_ve_vl_vsclot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssl : ClangBuiltin<"__builtin_ve_vl_vsclncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssml : ClangBuiltin<"__builtin_ve_vl_vsclncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>; +let TargetPrefix = "ve" in def int_ve_vl_andm_mmm : ClangBuiltin<"__builtin_ve_vl_andm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_andm_MMM : ClangBuiltin<"__builtin_ve_vl_andm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_orm_mmm : ClangBuiltin<"__builtin_ve_vl_orm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_orm_MMM : ClangBuiltin<"__builtin_ve_vl_orm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_xorm_mmm : ClangBuiltin<"__builtin_ve_vl_xorm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_xorm_MMM : ClangBuiltin<"__builtin_ve_vl_xorm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_eqvm_mmm : ClangBuiltin<"__builtin_ve_vl_eqvm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_eqvm_MMM : ClangBuiltin<"__builtin_ve_vl_eqvm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_nndm_mmm : ClangBuiltin<"__builtin_ve_vl_nndm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_nndm_MMM : ClangBuiltin<"__builtin_ve_vl_nndm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_negm_mm : ClangBuiltin<"__builtin_ve_vl_negm_mm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_negm_MM : ClangBuiltin<"__builtin_ve_vl_negm_MM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_pcvm_sml : ClangBuiltin<"__builtin_ve_vl_pcvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lzvm_sml : ClangBuiltin<"__builtin_ve_vl_lzvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_tovm_sml : ClangBuiltin<"__builtin_ve_vl_tovm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_lcr_sss : ClangBuiltin<"__builtin_ve_vl_lcr_sss">, Intrinsic<[LLVMType<i64>], [LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>; +let TargetPrefix = "ve" in def int_ve_vl_scr_sss : ClangBuiltin<"__builtin_ve_vl_scr_sss">, Intrinsic<[], [LLVMType<i64>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_tscr_ssss : ClangBuiltin<"__builtin_ve_vl_tscr_ssss">, Intrinsic<[LLVMType<i64>], [LLVMType<i64>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_fidcr_sss : ClangBuiltin<"__builtin_ve_vl_fidcr_sss">, Intrinsic<[LLVMType<i64>], [LLVMType<i64>, LLVMType<i32>], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_fencei : ClangBuiltin<"__builtin_ve_vl_fencei">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_fencem_s : ClangBuiltin<"__builtin_ve_vl_fencem_s">, Intrinsic<[], [LLVMType<i32>], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_fencec_s : ClangBuiltin<"__builtin_ve_vl_fencec_s">, Intrinsic<[], [LLVMType<i32>], [IntrNoMem, IntrHasSideEffects]>; +let TargetPrefix = "ve" in def int_ve_vl_svob : ClangBuiltin<"__builtin_ve_vl_svob">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td index aecc3d91fae7..f313be1b2235 100644 --- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td @@ -31,6 +31,10 @@ def int_wasm_memory_grow : Intrinsic<[llvm_anyint_ty], //===----------------------------------------------------------------------===// def int_wasm_ref_null_extern : Intrinsic<[llvm_externref_ty], [], [IntrNoMem]>; def int_wasm_ref_null_func : Intrinsic<[llvm_funcref_ty], [], [IntrNoMem]>; +def int_wasm_ref_is_null_extern : Intrinsic<[llvm_i32_ty], [llvm_externref_ty], + [IntrNoMem], "llvm.wasm.ref.is_null.extern">; +def int_wasm_ref_is_null_func : Intrinsic<[llvm_i32_ty], [llvm_funcref_ty], + [IntrNoMem], "llvm.wasm.ref.is_null.func">; //===----------------------------------------------------------------------===// // Table intrinsics @@ -256,16 +260,30 @@ def int_wasm_relaxed_trunc_unsigned: [llvm_v4f32_ty], [IntrNoMem, IntrSpeculatable]>; -def int_wasm_relaxed_trunc_zero_signed: +def int_wasm_relaxed_trunc_signed_zero: Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem, IntrSpeculatable]>; -def int_wasm_relaxed_trunc_zero_unsigned: +def int_wasm_relaxed_trunc_unsigned_zero: Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem, IntrSpeculatable]>; +def int_wasm_relaxed_q15mulr_signed: + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty], + [IntrNoMem, IntrSpeculatable]>; + +def int_wasm_dot_i8x16_i7x16_signed: + Intrinsic<[llvm_v8i16_ty], + [llvm_v16i8_ty, llvm_v16i8_ty], + [IntrNoMem, IntrSpeculatable]>; + +def int_wasm_dot_i8x16_i7x16_add_signed: + Intrinsic<[llvm_v4i32_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v4i32_ty], + [IntrNoMem, IntrSpeculatable]>; //===----------------------------------------------------------------------===// // Thread-local storage intrinsics diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td index 8de737a1c7a5..0930abcc0993 100644 --- a/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/llvm/include/llvm/IR/IntrinsicsX86.td @@ -31,20 +31,20 @@ let TargetPrefix = "x86" in { //===----------------------------------------------------------------------===// // FLAGS. let TargetPrefix = "x86" in { - def int_x86_flags_read_u32 : GCCBuiltin<"__builtin_ia32_readeflags_u32">, + def int_x86_flags_read_u32 : ClangBuiltin<"__builtin_ia32_readeflags_u32">, Intrinsic<[llvm_i32_ty], [], []>; - def int_x86_flags_read_u64 : GCCBuiltin<"__builtin_ia32_readeflags_u64">, + def int_x86_flags_read_u64 : ClangBuiltin<"__builtin_ia32_readeflags_u64">, Intrinsic<[llvm_i64_ty], [], []>; - def int_x86_flags_write_u32 : GCCBuiltin<"__builtin_ia32_writeeflags_u32">, + def int_x86_flags_write_u32 : ClangBuiltin<"__builtin_ia32_writeeflags_u32">, Intrinsic<[], [llvm_i32_ty], []>; - def int_x86_flags_write_u64 : GCCBuiltin<"__builtin_ia32_writeeflags_u64">, + def int_x86_flags_write_u64 : ClangBuiltin<"__builtin_ia32_writeeflags_u64">, Intrinsic<[], [llvm_i64_ty], []>; } //===----------------------------------------------------------------------===// // Read Time Stamp Counter. let TargetPrefix = "x86" in { - def int_x86_rdtsc : GCCBuiltin<"__builtin_ia32_rdtsc">, + def int_x86_rdtsc : ClangBuiltin<"__builtin_ia32_rdtsc">, Intrinsic<[llvm_i64_ty], [], []>; def int_x86_rdtscp : Intrinsic<[llvm_i64_ty, llvm_i32_ty], [], []>; @@ -52,42 +52,52 @@ let TargetPrefix = "x86" in { // Read Performance-Monitoring Counter. let TargetPrefix = "x86" in { - def int_x86_rdpmc : GCCBuiltin<"__builtin_ia32_rdpmc">, + def int_x86_rdpmc : ClangBuiltin<"__builtin_ia32_rdpmc">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty], []>; } // Read processor ID. let TargetPrefix = "x86" in { - def int_x86_rdpid : GCCBuiltin<"__builtin_ia32_rdpid">, + def int_x86_rdpid : ClangBuiltin<"__builtin_ia32_rdpid">, Intrinsic<[llvm_i32_ty], [], []>; } +// Lock bit test. +let TargetPrefix = "x86" in { + def int_x86_atomic_bts : Intrinsic<[llvm_anyint_ty], [llvm_ptr_ty, llvm_i8_ty], + [ImmArg<ArgIndex<1>>]>; + def int_x86_atomic_btc : Intrinsic<[llvm_anyint_ty], [llvm_ptr_ty, llvm_i8_ty], + [ImmArg<ArgIndex<1>>]>; + def int_x86_atomic_btr : Intrinsic<[llvm_anyint_ty], [llvm_ptr_ty, llvm_i8_ty], + [ImmArg<ArgIndex<1>>]>; +} + //===----------------------------------------------------------------------===// // CET SS let TargetPrefix = "x86" in { - def int_x86_incsspd : GCCBuiltin<"__builtin_ia32_incsspd">, + def int_x86_incsspd : ClangBuiltin<"__builtin_ia32_incsspd">, Intrinsic<[], [llvm_i32_ty], []>; - def int_x86_incsspq : GCCBuiltin<"__builtin_ia32_incsspq">, + def int_x86_incsspq : ClangBuiltin<"__builtin_ia32_incsspq">, Intrinsic<[], [llvm_i64_ty], []>; - def int_x86_rdsspd : GCCBuiltin<"__builtin_ia32_rdsspd">, + def int_x86_rdsspd : ClangBuiltin<"__builtin_ia32_rdsspd">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; - def int_x86_rdsspq : GCCBuiltin<"__builtin_ia32_rdsspq">, + def int_x86_rdsspq : ClangBuiltin<"__builtin_ia32_rdsspq">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty], []>; - def int_x86_saveprevssp : GCCBuiltin<"__builtin_ia32_saveprevssp">, + def int_x86_saveprevssp : ClangBuiltin<"__builtin_ia32_saveprevssp">, Intrinsic<[], [], []>; - def int_x86_rstorssp : GCCBuiltin<"__builtin_ia32_rstorssp">, + def int_x86_rstorssp : ClangBuiltin<"__builtin_ia32_rstorssp">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_wrssd : GCCBuiltin<"__builtin_ia32_wrssd">, + def int_x86_wrssd : ClangBuiltin<"__builtin_ia32_wrssd">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; - def int_x86_wrssq : GCCBuiltin<"__builtin_ia32_wrssq">, + def int_x86_wrssq : ClangBuiltin<"__builtin_ia32_wrssq">, Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>; - def int_x86_wrussd : GCCBuiltin<"__builtin_ia32_wrussd">, + def int_x86_wrussd : ClangBuiltin<"__builtin_ia32_wrussd">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; - def int_x86_wrussq : GCCBuiltin<"__builtin_ia32_wrussq">, + def int_x86_wrussq : ClangBuiltin<"__builtin_ia32_wrussq">, Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>; - def int_x86_setssbsy : GCCBuiltin<"__builtin_ia32_setssbsy">, + def int_x86_setssbsy : ClangBuiltin<"__builtin_ia32_setssbsy">, Intrinsic<[], [], []>; - def int_x86_clrssbsy : GCCBuiltin<"__builtin_ia32_clrssbsy">, + def int_x86_clrssbsy : ClangBuiltin<"__builtin_ia32_clrssbsy">, Intrinsic<[], [llvm_ptr_ty], []>; } @@ -95,57 +105,57 @@ let TargetPrefix = "x86" in { // 3DNow! let TargetPrefix = "x86" in { - def int_x86_3dnow_pavgusb : GCCBuiltin<"__builtin_ia32_pavgusb">, + def int_x86_3dnow_pavgusb : ClangBuiltin<"__builtin_ia32_pavgusb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pf2id : GCCBuiltin<"__builtin_ia32_pf2id">, + def int_x86_3dnow_pf2id : ClangBuiltin<"__builtin_ia32_pf2id">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfacc : GCCBuiltin<"__builtin_ia32_pfacc">, + def int_x86_3dnow_pfacc : ClangBuiltin<"__builtin_ia32_pfacc">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfadd : GCCBuiltin<"__builtin_ia32_pfadd">, + def int_x86_3dnow_pfadd : ClangBuiltin<"__builtin_ia32_pfadd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfcmpeq : GCCBuiltin<"__builtin_ia32_pfcmpeq">, + def int_x86_3dnow_pfcmpeq : ClangBuiltin<"__builtin_ia32_pfcmpeq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfcmpge : GCCBuiltin<"__builtin_ia32_pfcmpge">, + def int_x86_3dnow_pfcmpge : ClangBuiltin<"__builtin_ia32_pfcmpge">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfcmpgt : GCCBuiltin<"__builtin_ia32_pfcmpgt">, + def int_x86_3dnow_pfcmpgt : ClangBuiltin<"__builtin_ia32_pfcmpgt">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfmax : GCCBuiltin<"__builtin_ia32_pfmax">, + def int_x86_3dnow_pfmax : ClangBuiltin<"__builtin_ia32_pfmax">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfmin : GCCBuiltin<"__builtin_ia32_pfmin">, + def int_x86_3dnow_pfmin : ClangBuiltin<"__builtin_ia32_pfmin">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfmul : GCCBuiltin<"__builtin_ia32_pfmul">, + def int_x86_3dnow_pfmul : ClangBuiltin<"__builtin_ia32_pfmul">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfrcp : GCCBuiltin<"__builtin_ia32_pfrcp">, + def int_x86_3dnow_pfrcp : ClangBuiltin<"__builtin_ia32_pfrcp">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfrcpit1 : GCCBuiltin<"__builtin_ia32_pfrcpit1">, + def int_x86_3dnow_pfrcpit1 : ClangBuiltin<"__builtin_ia32_pfrcpit1">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfrcpit2 : GCCBuiltin<"__builtin_ia32_pfrcpit2">, + def int_x86_3dnow_pfrcpit2 : ClangBuiltin<"__builtin_ia32_pfrcpit2">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfrsqrt : GCCBuiltin<"__builtin_ia32_pfrsqrt">, + def int_x86_3dnow_pfrsqrt : ClangBuiltin<"__builtin_ia32_pfrsqrt">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfrsqit1 : GCCBuiltin<"__builtin_ia32_pfrsqit1">, + def int_x86_3dnow_pfrsqit1 : ClangBuiltin<"__builtin_ia32_pfrsqit1">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfsub : GCCBuiltin<"__builtin_ia32_pfsub">, + def int_x86_3dnow_pfsub : ClangBuiltin<"__builtin_ia32_pfsub">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pfsubr : GCCBuiltin<"__builtin_ia32_pfsubr">, + def int_x86_3dnow_pfsubr : ClangBuiltin<"__builtin_ia32_pfsubr">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pi2fd : GCCBuiltin<"__builtin_ia32_pi2fd">, + def int_x86_3dnow_pi2fd : ClangBuiltin<"__builtin_ia32_pi2fd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnow_pmulhrw : GCCBuiltin<"__builtin_ia32_pmulhrw">, + def int_x86_3dnow_pmulhrw : ClangBuiltin<"__builtin_ia32_pmulhrw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; } @@ -154,15 +164,15 @@ let TargetPrefix = "x86" in { // 3DNow! extensions let TargetPrefix = "x86" in { - def int_x86_3dnowa_pf2iw : GCCBuiltin<"__builtin_ia32_pf2iw">, + def int_x86_3dnowa_pf2iw : ClangBuiltin<"__builtin_ia32_pf2iw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnowa_pfnacc : GCCBuiltin<"__builtin_ia32_pfnacc">, + def int_x86_3dnowa_pfnacc : ClangBuiltin<"__builtin_ia32_pfnacc">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnowa_pfpnacc : GCCBuiltin<"__builtin_ia32_pfpnacc">, + def int_x86_3dnowa_pfpnacc : ClangBuiltin<"__builtin_ia32_pfpnacc">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_3dnowa_pi2fw : GCCBuiltin<"__builtin_ia32_pi2fw">, + def int_x86_3dnowa_pi2fw : ClangBuiltin<"__builtin_ia32_pi2fw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; def int_x86_3dnowa_pswapd : Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; @@ -173,35 +183,35 @@ let TargetPrefix = "x86" in { // Arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_rcp_ss : GCCBuiltin<"__builtin_ia32_rcpss">, + def int_x86_sse_rcp_ss : ClangBuiltin<"__builtin_ia32_rcpss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_rcp_ps : GCCBuiltin<"__builtin_ia32_rcpps">, + def int_x86_sse_rcp_ps : ClangBuiltin<"__builtin_ia32_rcpps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_rsqrt_ss : GCCBuiltin<"__builtin_ia32_rsqrtss">, + def int_x86_sse_rsqrt_ss : ClangBuiltin<"__builtin_ia32_rsqrtss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_rsqrt_ps : GCCBuiltin<"__builtin_ia32_rsqrtps">, + def int_x86_sse_rsqrt_ps : ClangBuiltin<"__builtin_ia32_rsqrtps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_min_ss : GCCBuiltin<"__builtin_ia32_minss">, + def int_x86_sse_min_ss : ClangBuiltin<"__builtin_ia32_minss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_min_ps : GCCBuiltin<"__builtin_ia32_minps">, + def int_x86_sse_min_ps : ClangBuiltin<"__builtin_ia32_minps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_max_ss : GCCBuiltin<"__builtin_ia32_maxss">, + def int_x86_sse_max_ss : ClangBuiltin<"__builtin_ia32_maxss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_max_ps : GCCBuiltin<"__builtin_ia32_maxps">, + def int_x86_sse_max_ps : ClangBuiltin<"__builtin_ia32_maxps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; } // Comparison ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_cmp_ss : GCCBuiltin<"__builtin_ia32_cmpss">, + def int_x86_sse_cmp_ss : ClangBuiltin<"__builtin_ia32_cmpss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; // NOTE: This comparison intrinsic is not used by clang as long as the @@ -209,40 +219,40 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse_cmp_ps : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse_comieq_ss : GCCBuiltin<"__builtin_ia32_comieq">, + def int_x86_sse_comieq_ss : ClangBuiltin<"__builtin_ia32_comieq">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_comilt_ss : GCCBuiltin<"__builtin_ia32_comilt">, + def int_x86_sse_comilt_ss : ClangBuiltin<"__builtin_ia32_comilt">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_comile_ss : GCCBuiltin<"__builtin_ia32_comile">, + def int_x86_sse_comile_ss : ClangBuiltin<"__builtin_ia32_comile">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_comigt_ss : GCCBuiltin<"__builtin_ia32_comigt">, + def int_x86_sse_comigt_ss : ClangBuiltin<"__builtin_ia32_comigt">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_comige_ss : GCCBuiltin<"__builtin_ia32_comige">, + def int_x86_sse_comige_ss : ClangBuiltin<"__builtin_ia32_comige">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_comineq_ss : GCCBuiltin<"__builtin_ia32_comineq">, + def int_x86_sse_comineq_ss : ClangBuiltin<"__builtin_ia32_comineq">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomieq_ss : GCCBuiltin<"__builtin_ia32_ucomieq">, + def int_x86_sse_ucomieq_ss : ClangBuiltin<"__builtin_ia32_ucomieq">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomilt_ss : GCCBuiltin<"__builtin_ia32_ucomilt">, + def int_x86_sse_ucomilt_ss : ClangBuiltin<"__builtin_ia32_ucomilt">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomile_ss : GCCBuiltin<"__builtin_ia32_ucomile">, + def int_x86_sse_ucomile_ss : ClangBuiltin<"__builtin_ia32_ucomile">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomigt_ss : GCCBuiltin<"__builtin_ia32_ucomigt">, + def int_x86_sse_ucomigt_ss : ClangBuiltin<"__builtin_ia32_ucomigt">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomige_ss : GCCBuiltin<"__builtin_ia32_ucomige">, + def int_x86_sse_ucomige_ss : ClangBuiltin<"__builtin_ia32_ucomige">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_ucomineq_ss : GCCBuiltin<"__builtin_ia32_ucomineq">, + def int_x86_sse_ucomineq_ss : ClangBuiltin<"__builtin_ia32_ucomineq">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; } @@ -250,27 +260,27 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_cvtss2si : GCCBuiltin<"__builtin_ia32_cvtss2si">, + def int_x86_sse_cvtss2si : ClangBuiltin<"__builtin_ia32_cvtss2si">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvtss2si64 : GCCBuiltin<"__builtin_ia32_cvtss2si64">, + def int_x86_sse_cvtss2si64 : ClangBuiltin<"__builtin_ia32_cvtss2si64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvttss2si : GCCBuiltin<"__builtin_ia32_cvttss2si">, + def int_x86_sse_cvttss2si : ClangBuiltin<"__builtin_ia32_cvttss2si">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvttss2si64 : GCCBuiltin<"__builtin_ia32_cvttss2si64">, + def int_x86_sse_cvttss2si64 : ClangBuiltin<"__builtin_ia32_cvttss2si64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvtps2pi : GCCBuiltin<"__builtin_ia32_cvtps2pi">, + def int_x86_sse_cvtps2pi : ClangBuiltin<"__builtin_ia32_cvtps2pi">, Intrinsic<[llvm_x86mmx_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvttps2pi: GCCBuiltin<"__builtin_ia32_cvttps2pi">, + def int_x86_sse_cvttps2pi: ClangBuiltin<"__builtin_ia32_cvttps2pi">, Intrinsic<[llvm_x86mmx_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_cvtpi2ps : GCCBuiltin<"__builtin_ia32_cvtpi2ps">, + def int_x86_sse_cvtpi2ps : ClangBuiltin<"__builtin_ia32_cvtpi2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_x86mmx_ty], [IntrNoMem]>; } // Cacheability support ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_sfence : GCCBuiltin<"__builtin_ia32_sfence">, + def int_x86_sse_sfence : ClangBuiltin<"__builtin_ia32_sfence">, Intrinsic<[], [], []>; } @@ -291,7 +301,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Misc. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_movmsk_ps : GCCBuiltin<"__builtin_ia32_movmskps">, + def int_x86_sse_movmsk_ps : ClangBuiltin<"__builtin_ia32_movmskps">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; } @@ -300,23 +310,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FP arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_min_sd : GCCBuiltin<"__builtin_ia32_minsd">, + def int_x86_sse2_min_sd : ClangBuiltin<"__builtin_ia32_minsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_min_pd : GCCBuiltin<"__builtin_ia32_minpd">, + def int_x86_sse2_min_pd : ClangBuiltin<"__builtin_ia32_minpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_max_sd : GCCBuiltin<"__builtin_ia32_maxsd">, + def int_x86_sse2_max_sd : ClangBuiltin<"__builtin_ia32_maxsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_max_pd : GCCBuiltin<"__builtin_ia32_maxpd">, + def int_x86_sse2_max_pd : ClangBuiltin<"__builtin_ia32_maxpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; } // FP comparison ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_cmp_sd : GCCBuiltin<"__builtin_ia32_cmpsd">, + def int_x86_sse2_cmp_sd : ClangBuiltin<"__builtin_ia32_cmpsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; // NOTE: This comparison intrinsic is not used by clang as long as the @@ -324,176 +334,176 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_cmp_pd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse2_comieq_sd : GCCBuiltin<"__builtin_ia32_comisdeq">, + def int_x86_sse2_comieq_sd : ClangBuiltin<"__builtin_ia32_comisdeq">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_comilt_sd : GCCBuiltin<"__builtin_ia32_comisdlt">, + def int_x86_sse2_comilt_sd : ClangBuiltin<"__builtin_ia32_comisdlt">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_comile_sd : GCCBuiltin<"__builtin_ia32_comisdle">, + def int_x86_sse2_comile_sd : ClangBuiltin<"__builtin_ia32_comisdle">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_comigt_sd : GCCBuiltin<"__builtin_ia32_comisdgt">, + def int_x86_sse2_comigt_sd : ClangBuiltin<"__builtin_ia32_comisdgt">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_comige_sd : GCCBuiltin<"__builtin_ia32_comisdge">, + def int_x86_sse2_comige_sd : ClangBuiltin<"__builtin_ia32_comisdge">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_comineq_sd : GCCBuiltin<"__builtin_ia32_comisdneq">, + def int_x86_sse2_comineq_sd : ClangBuiltin<"__builtin_ia32_comisdneq">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomieq_sd : GCCBuiltin<"__builtin_ia32_ucomisdeq">, + def int_x86_sse2_ucomieq_sd : ClangBuiltin<"__builtin_ia32_ucomisdeq">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomilt_sd : GCCBuiltin<"__builtin_ia32_ucomisdlt">, + def int_x86_sse2_ucomilt_sd : ClangBuiltin<"__builtin_ia32_ucomisdlt">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomile_sd : GCCBuiltin<"__builtin_ia32_ucomisdle">, + def int_x86_sse2_ucomile_sd : ClangBuiltin<"__builtin_ia32_ucomisdle">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomigt_sd : GCCBuiltin<"__builtin_ia32_ucomisdgt">, + def int_x86_sse2_ucomigt_sd : ClangBuiltin<"__builtin_ia32_ucomisdgt">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomige_sd : GCCBuiltin<"__builtin_ia32_ucomisdge">, + def int_x86_sse2_ucomige_sd : ClangBuiltin<"__builtin_ia32_ucomisdge">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_ucomineq_sd : GCCBuiltin<"__builtin_ia32_ucomisdneq">, + def int_x86_sse2_ucomineq_sd : ClangBuiltin<"__builtin_ia32_ucomisdneq">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; } // Integer arithmetic ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw128">, + def int_x86_sse2_pmulhu_w : ClangBuiltin<"__builtin_ia32_pmulhuw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw128">, + def int_x86_sse2_pmulh_w : ClangBuiltin<"__builtin_ia32_pmulhw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd128">, + def int_x86_sse2_pmadd_wd : ClangBuiltin<"__builtin_ia32_pmaddwd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb128">, + def int_x86_sse2_pavg_b : ClangBuiltin<"__builtin_ia32_pavgb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw128">, + def int_x86_sse2_pavg_w : ClangBuiltin<"__builtin_ia32_pavgw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw128">, + def int_x86_sse2_psad_bw : ClangBuiltin<"__builtin_ia32_psadbw128">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, Commutative]>; } // Integer shift ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_psll_w : GCCBuiltin<"__builtin_ia32_psllw128">, + def int_x86_sse2_psll_w : ClangBuiltin<"__builtin_ia32_psllw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_sse2_psll_d : GCCBuiltin<"__builtin_ia32_pslld128">, + def int_x86_sse2_psll_d : ClangBuiltin<"__builtin_ia32_pslld128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sse2_psll_q : GCCBuiltin<"__builtin_ia32_psllq128">, + def int_x86_sse2_psll_q : ClangBuiltin<"__builtin_ia32_psllq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_sse2_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw128">, + def int_x86_sse2_psrl_w : ClangBuiltin<"__builtin_ia32_psrlw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_sse2_psrl_d : GCCBuiltin<"__builtin_ia32_psrld128">, + def int_x86_sse2_psrl_d : ClangBuiltin<"__builtin_ia32_psrld128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq128">, + def int_x86_sse2_psrl_q : ClangBuiltin<"__builtin_ia32_psrlq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_sse2_psra_w : GCCBuiltin<"__builtin_ia32_psraw128">, + def int_x86_sse2_psra_w : ClangBuiltin<"__builtin_ia32_psraw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_sse2_psra_d : GCCBuiltin<"__builtin_ia32_psrad128">, + def int_x86_sse2_psra_d : ClangBuiltin<"__builtin_ia32_psrad128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; // Oddly these don't require an immediate due to a gcc compatibility issue. - def int_x86_sse2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi128">, + def int_x86_sse2_pslli_w : ClangBuiltin<"__builtin_ia32_psllwi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi128">, + def int_x86_sse2_pslli_d : ClangBuiltin<"__builtin_ia32_pslldi128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi128">, + def int_x86_sse2_pslli_q : ClangBuiltin<"__builtin_ia32_psllqi128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi128">, + def int_x86_sse2_psrli_w : ClangBuiltin<"__builtin_ia32_psrlwi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi128">, + def int_x86_sse2_psrli_d : ClangBuiltin<"__builtin_ia32_psrldi128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi128">, + def int_x86_sse2_psrli_q : ClangBuiltin<"__builtin_ia32_psrlqi128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi128">, + def int_x86_sse2_psrai_w : ClangBuiltin<"__builtin_ia32_psrawi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse2_psrai_d : GCCBuiltin<"__builtin_ia32_psradi128">, + def int_x86_sse2_psrai_d : ClangBuiltin<"__builtin_ia32_psradi128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; } // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_cvtpd2dq : GCCBuiltin<"__builtin_ia32_cvtpd2dq">, + def int_x86_sse2_cvtpd2dq : ClangBuiltin<"__builtin_ia32_cvtpd2dq">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvttpd2dq : GCCBuiltin<"__builtin_ia32_cvttpd2dq">, + def int_x86_sse2_cvttpd2dq : ClangBuiltin<"__builtin_ia32_cvttpd2dq">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvtpd2ps : GCCBuiltin<"__builtin_ia32_cvtpd2ps">, + def int_x86_sse2_cvtpd2ps : ClangBuiltin<"__builtin_ia32_cvtpd2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvtps2dq : GCCBuiltin<"__builtin_ia32_cvtps2dq">, + def int_x86_sse2_cvtps2dq : ClangBuiltin<"__builtin_ia32_cvtps2dq">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse2_cvttps2dq : GCCBuiltin<"__builtin_ia32_cvttps2dq">, + def int_x86_sse2_cvttps2dq : ClangBuiltin<"__builtin_ia32_cvttps2dq">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse2_cvtsd2si : GCCBuiltin<"__builtin_ia32_cvtsd2si">, + def int_x86_sse2_cvtsd2si : ClangBuiltin<"__builtin_ia32_cvtsd2si">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvtsd2si64 : GCCBuiltin<"__builtin_ia32_cvtsd2si64">, + def int_x86_sse2_cvtsd2si64 : ClangBuiltin<"__builtin_ia32_cvtsd2si64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvttsd2si : GCCBuiltin<"__builtin_ia32_cvttsd2si">, + def int_x86_sse2_cvttsd2si : ClangBuiltin<"__builtin_ia32_cvttsd2si">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_cvttsd2si64">, + def int_x86_sse2_cvttsd2si64 : ClangBuiltin<"__builtin_ia32_cvttsd2si64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_cvtsd2ss : GCCBuiltin<"__builtin_ia32_cvtsd2ss">, + def int_x86_sse2_cvtsd2ss : ClangBuiltin<"__builtin_ia32_cvtsd2ss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse_cvtpd2pi : GCCBuiltin<"__builtin_ia32_cvtpd2pi">, + def int_x86_sse_cvtpd2pi : ClangBuiltin<"__builtin_ia32_cvtpd2pi">, Intrinsic<[llvm_x86mmx_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse_cvttpd2pi: GCCBuiltin<"__builtin_ia32_cvttpd2pi">, + def int_x86_sse_cvttpd2pi: ClangBuiltin<"__builtin_ia32_cvttpd2pi">, Intrinsic<[llvm_x86mmx_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse_cvtpi2pd : GCCBuiltin<"__builtin_ia32_cvtpi2pd">, + def int_x86_sse_cvtpi2pd : ClangBuiltin<"__builtin_ia32_cvtpi2pd">, Intrinsic<[llvm_v2f64_ty], [llvm_x86mmx_ty], [IntrNoMem]>; } // Misc. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_packsswb_128 : GCCBuiltin<"__builtin_ia32_packsswb128">, + def int_x86_sse2_packsswb_128 : ClangBuiltin<"__builtin_ia32_packsswb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_sse2_packssdw_128 : GCCBuiltin<"__builtin_ia32_packssdw128">, + def int_x86_sse2_packssdw_128 : ClangBuiltin<"__builtin_ia32_packssdw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sse2_packuswb_128 : GCCBuiltin<"__builtin_ia32_packuswb128">, + def int_x86_sse2_packuswb_128 : ClangBuiltin<"__builtin_ia32_packuswb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_sse2_movmsk_pd : GCCBuiltin<"__builtin_ia32_movmskpd">, + def int_x86_sse2_movmsk_pd : ClangBuiltin<"__builtin_ia32_movmskpd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_pmovmskb_128 : GCCBuiltin<"__builtin_ia32_pmovmskb128">, + def int_x86_sse2_pmovmskb_128 : ClangBuiltin<"__builtin_ia32_pmovmskb128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_sse2_maskmov_dqu : GCCBuiltin<"__builtin_ia32_maskmovdqu">, + def int_x86_sse2_maskmov_dqu : ClangBuiltin<"__builtin_ia32_maskmovdqu">, Intrinsic<[], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_ptr_ty], []>; - def int_x86_sse2_clflush : GCCBuiltin<"__builtin_ia32_clflush">, + def int_x86_sse2_clflush : ClangBuiltin<"__builtin_ia32_clflush">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_sse2_lfence : GCCBuiltin<"__builtin_ia32_lfence">, + def int_x86_sse2_lfence : ClangBuiltin<"__builtin_ia32_lfence">, Intrinsic<[], [], []>; - def int_x86_sse2_mfence : GCCBuiltin<"__builtin_ia32_mfence">, + def int_x86_sse2_mfence : ClangBuiltin<"__builtin_ia32_mfence">, Intrinsic<[], [], []>; - def int_x86_sse2_pause : GCCBuiltin<"__builtin_ia32_pause">, + def int_x86_sse2_pause : ClangBuiltin<"__builtin_ia32_pause">, Intrinsic<[], [], []>; } @@ -502,42 +512,42 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Addition / subtraction ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse3_addsub_ps : GCCBuiltin<"__builtin_ia32_addsubps">, + def int_x86_sse3_addsub_ps : ClangBuiltin<"__builtin_ia32_addsubps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse3_addsub_pd : GCCBuiltin<"__builtin_ia32_addsubpd">, + def int_x86_sse3_addsub_pd : ClangBuiltin<"__builtin_ia32_addsubpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; } // Horizontal ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse3_hadd_ps : GCCBuiltin<"__builtin_ia32_haddps">, + def int_x86_sse3_hadd_ps : ClangBuiltin<"__builtin_ia32_haddps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse3_hadd_pd : GCCBuiltin<"__builtin_ia32_haddpd">, + def int_x86_sse3_hadd_pd : ClangBuiltin<"__builtin_ia32_haddpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse3_hsub_ps : GCCBuiltin<"__builtin_ia32_hsubps">, + def int_x86_sse3_hsub_ps : ClangBuiltin<"__builtin_ia32_hsubps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse3_hsub_pd : GCCBuiltin<"__builtin_ia32_hsubpd">, + def int_x86_sse3_hsub_pd : ClangBuiltin<"__builtin_ia32_hsubpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; } // Specialized unaligned load. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse3_ldu_dq : GCCBuiltin<"__builtin_ia32_lddqu">, + def int_x86_sse3_ldu_dq : ClangBuiltin<"__builtin_ia32_lddqu">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty], [IntrReadMem]>; } // Thread synchronization ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse3_monitor : GCCBuiltin<"__builtin_ia32_monitor">, + def int_x86_sse3_monitor : ClangBuiltin<"__builtin_ia32_monitor">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; - def int_x86_sse3_mwait : GCCBuiltin<"__builtin_ia32_mwait">, + def int_x86_sse3_mwait : ClangBuiltin<"__builtin_ia32_mwait">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], []>; } @@ -547,112 +557,112 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Horizontal arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_ssse3_phadd_w : GCCBuiltin<"__builtin_ia32_phaddw">, + def int_x86_ssse3_phadd_w : ClangBuiltin<"__builtin_ia32_phaddw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phadd_w_128 : GCCBuiltin<"__builtin_ia32_phaddw128">, + def int_x86_ssse3_phadd_w_128 : ClangBuiltin<"__builtin_ia32_phaddw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_ssse3_phadd_d : GCCBuiltin<"__builtin_ia32_phaddd">, + def int_x86_ssse3_phadd_d : ClangBuiltin<"__builtin_ia32_phaddd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phadd_d_128 : GCCBuiltin<"__builtin_ia32_phaddd128">, + def int_x86_ssse3_phadd_d_128 : ClangBuiltin<"__builtin_ia32_phaddd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_ssse3_phadd_sw : GCCBuiltin<"__builtin_ia32_phaddsw">, + def int_x86_ssse3_phadd_sw : ClangBuiltin<"__builtin_ia32_phaddsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phadd_sw_128 : GCCBuiltin<"__builtin_ia32_phaddsw128">, + def int_x86_ssse3_phadd_sw_128 : ClangBuiltin<"__builtin_ia32_phaddsw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_w : GCCBuiltin<"__builtin_ia32_phsubw">, + def int_x86_ssse3_phsub_w : ClangBuiltin<"__builtin_ia32_phsubw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_w_128 : GCCBuiltin<"__builtin_ia32_phsubw128">, + def int_x86_ssse3_phsub_w_128 : ClangBuiltin<"__builtin_ia32_phsubw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_d : GCCBuiltin<"__builtin_ia32_phsubd">, + def int_x86_ssse3_phsub_d : ClangBuiltin<"__builtin_ia32_phsubd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_d_128 : GCCBuiltin<"__builtin_ia32_phsubd128">, + def int_x86_ssse3_phsub_d_128 : ClangBuiltin<"__builtin_ia32_phsubd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_sw : GCCBuiltin<"__builtin_ia32_phsubsw">, + def int_x86_ssse3_phsub_sw : ClangBuiltin<"__builtin_ia32_phsubsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_phsub_sw_128 : GCCBuiltin<"__builtin_ia32_phsubsw128">, + def int_x86_ssse3_phsub_sw_128 : ClangBuiltin<"__builtin_ia32_phsubsw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_ssse3_pmadd_ub_sw : GCCBuiltin<"__builtin_ia32_pmaddubsw">, + def int_x86_ssse3_pmadd_ub_sw : ClangBuiltin<"__builtin_ia32_pmaddubsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pmadd_ub_sw_128 : GCCBuiltin<"__builtin_ia32_pmaddubsw128">, + def int_x86_ssse3_pmadd_ub_sw_128 : ClangBuiltin<"__builtin_ia32_pmaddubsw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; } // Packed multiply high with round and scale let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_ssse3_pmul_hr_sw : GCCBuiltin<"__builtin_ia32_pmulhrsw">, + def int_x86_ssse3_pmul_hr_sw : ClangBuiltin<"__builtin_ia32_pmulhrsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_ssse3_pmul_hr_sw_128 : GCCBuiltin<"__builtin_ia32_pmulhrsw128">, + def int_x86_ssse3_pmul_hr_sw_128 : ClangBuiltin<"__builtin_ia32_pmulhrsw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; } // Shuffle ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_ssse3_pshuf_b : GCCBuiltin<"__builtin_ia32_pshufb">, + def int_x86_ssse3_pshuf_b : ClangBuiltin<"__builtin_ia32_pshufb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pshuf_b_128 : GCCBuiltin<"__builtin_ia32_pshufb128">, + def int_x86_ssse3_pshuf_b_128 : ClangBuiltin<"__builtin_ia32_pshufb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_sse_pshuf_w : GCCBuiltin<"__builtin_ia32_pshufw">, + def int_x86_sse_pshuf_w : ClangBuiltin<"__builtin_ia32_pshufw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } // Sign ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_ssse3_psign_b : GCCBuiltin<"__builtin_ia32_psignb">, + def int_x86_ssse3_psign_b : ClangBuiltin<"__builtin_ia32_psignb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_psign_b_128 : GCCBuiltin<"__builtin_ia32_psignb128">, + def int_x86_ssse3_psign_b_128 : ClangBuiltin<"__builtin_ia32_psignb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_ssse3_psign_w : GCCBuiltin<"__builtin_ia32_psignw">, + def int_x86_ssse3_psign_w : ClangBuiltin<"__builtin_ia32_psignw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_psign_w_128 : GCCBuiltin<"__builtin_ia32_psignw128">, + def int_x86_ssse3_psign_w_128 : ClangBuiltin<"__builtin_ia32_psignw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_ssse3_psign_d : GCCBuiltin<"__builtin_ia32_psignd">, + def int_x86_ssse3_psign_d : ClangBuiltin<"__builtin_ia32_psignd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_psign_d_128 : GCCBuiltin<"__builtin_ia32_psignd128">, + def int_x86_ssse3_psign_d_128 : ClangBuiltin<"__builtin_ia32_psignd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; } // Absolute value ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_ssse3_pabs_b : GCCBuiltin<"__builtin_ia32_pabsb">, + def int_x86_ssse3_pabs_b : ClangBuiltin<"__builtin_ia32_pabsb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pabs_w : GCCBuiltin<"__builtin_ia32_pabsw">, + def int_x86_ssse3_pabs_w : ClangBuiltin<"__builtin_ia32_pabsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pabs_d : GCCBuiltin<"__builtin_ia32_pabsd">, + def int_x86_ssse3_pabs_d : ClangBuiltin<"__builtin_ia32_pabsd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; } @@ -661,149 +671,149 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FP rounding ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_round_ss : GCCBuiltin<"__builtin_ia32_roundss">, + def int_x86_sse41_round_ss : ClangBuiltin<"__builtin_ia32_roundss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse41_round_ps : GCCBuiltin<"__builtin_ia32_roundps">, + def int_x86_sse41_round_ps : ClangBuiltin<"__builtin_ia32_roundps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_sse41_round_sd : GCCBuiltin<"__builtin_ia32_roundsd">, + def int_x86_sse41_round_sd : ClangBuiltin<"__builtin_ia32_roundsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse41_round_pd : GCCBuiltin<"__builtin_ia32_roundpd">, + def int_x86_sse41_round_pd : ClangBuiltin<"__builtin_ia32_roundpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } // Vector min element let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_phminposuw : GCCBuiltin<"__builtin_ia32_phminposuw128">, + def int_x86_sse41_phminposuw : ClangBuiltin<"__builtin_ia32_phminposuw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; } // Advanced Encryption Standard (AES) Instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_aesni_aesimc : GCCBuiltin<"__builtin_ia32_aesimc128">, + def int_x86_aesni_aesimc : ClangBuiltin<"__builtin_ia32_aesimc128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesenc : GCCBuiltin<"__builtin_ia32_aesenc128">, + def int_x86_aesni_aesenc : ClangBuiltin<"__builtin_ia32_aesenc128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesenc_256 : GCCBuiltin<"__builtin_ia32_aesenc256">, + def int_x86_aesni_aesenc_256 : ClangBuiltin<"__builtin_ia32_aesenc256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesenc_512 : GCCBuiltin<"__builtin_ia32_aesenc512">, + def int_x86_aesni_aesenc_512 : ClangBuiltin<"__builtin_ia32_aesenc512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesenclast : GCCBuiltin<"__builtin_ia32_aesenclast128">, + def int_x86_aesni_aesenclast : ClangBuiltin<"__builtin_ia32_aesenclast128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_aesni_aesenclast_256 : - GCCBuiltin<"__builtin_ia32_aesenclast256">, + ClangBuiltin<"__builtin_ia32_aesenclast256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_aesni_aesenclast_512 : - GCCBuiltin<"__builtin_ia32_aesenclast512">, + ClangBuiltin<"__builtin_ia32_aesenclast512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesdec : GCCBuiltin<"__builtin_ia32_aesdec128">, + def int_x86_aesni_aesdec : ClangBuiltin<"__builtin_ia32_aesdec128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesdec_256 : GCCBuiltin<"__builtin_ia32_aesdec256">, + def int_x86_aesni_aesdec_256 : ClangBuiltin<"__builtin_ia32_aesdec256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesdec_512 : GCCBuiltin<"__builtin_ia32_aesdec512">, + def int_x86_aesni_aesdec_512 : ClangBuiltin<"__builtin_ia32_aesdec512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_aesni_aesdeclast : GCCBuiltin<"__builtin_ia32_aesdeclast128">, + def int_x86_aesni_aesdeclast : ClangBuiltin<"__builtin_ia32_aesdeclast128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_aesni_aesdeclast_256 : - GCCBuiltin<"__builtin_ia32_aesdeclast256">, + ClangBuiltin<"__builtin_ia32_aesdeclast256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_aesni_aesdeclast_512 : - GCCBuiltin<"__builtin_ia32_aesdeclast512">, + ClangBuiltin<"__builtin_ia32_aesdeclast512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; def int_x86_aesni_aeskeygenassist : - GCCBuiltin<"__builtin_ia32_aeskeygenassist128">, + ClangBuiltin<"__builtin_ia32_aeskeygenassist128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } // PCLMUL instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_pclmulqdq : GCCBuiltin<"__builtin_ia32_pclmulqdq128">, + def int_x86_pclmulqdq : ClangBuiltin<"__builtin_ia32_pclmulqdq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_pclmulqdq_256 : GCCBuiltin<"__builtin_ia32_pclmulqdq256">, + def int_x86_pclmulqdq_256 : ClangBuiltin<"__builtin_ia32_pclmulqdq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_pclmulqdq_512 : GCCBuiltin<"__builtin_ia32_pclmulqdq512">, + def int_x86_pclmulqdq_512 : ClangBuiltin<"__builtin_ia32_pclmulqdq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } // Vector pack let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_packusdw : GCCBuiltin<"__builtin_ia32_packusdw128">, + def int_x86_sse41_packusdw : ClangBuiltin<"__builtin_ia32_packusdw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; } // Vector insert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_insertps : GCCBuiltin<"__builtin_ia32_insertps128">, + def int_x86_sse41_insertps : ClangBuiltin<"__builtin_ia32_insertps128">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } // Vector blend let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_pblendvb : GCCBuiltin<"__builtin_ia32_pblendvb128">, + def int_x86_sse41_pblendvb : ClangBuiltin<"__builtin_ia32_pblendvb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_sse41_blendvpd : GCCBuiltin<"__builtin_ia32_blendvpd">, + def int_x86_sse41_blendvpd : ClangBuiltin<"__builtin_ia32_blendvpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse41_blendvps : GCCBuiltin<"__builtin_ia32_blendvps">, + def int_x86_sse41_blendvps : ClangBuiltin<"__builtin_ia32_blendvps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,llvm_v4f32_ty], [IntrNoMem]>; } // Vector dot product let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_dppd : GCCBuiltin<"__builtin_ia32_dppd">, + def int_x86_sse41_dppd : ClangBuiltin<"__builtin_ia32_dppd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, Commutative, ImmArg<ArgIndex<2>>]>; - def int_x86_sse41_dpps : GCCBuiltin<"__builtin_ia32_dpps">, + def int_x86_sse41_dpps : ClangBuiltin<"__builtin_ia32_dpps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, Commutative, ImmArg<ArgIndex<2>>]>; } // Vector sum of absolute differences let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw128">, + def int_x86_sse41_mpsadbw : ClangBuiltin<"__builtin_ia32_mpsadbw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } // Test instruction with bitwise comparison. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_ptestz : GCCBuiltin<"__builtin_ia32_ptestz128">, + def int_x86_sse41_ptestz : ClangBuiltin<"__builtin_ia32_ptestz128">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_sse41_ptestc : GCCBuiltin<"__builtin_ia32_ptestc128">, + def int_x86_sse41_ptestc : ClangBuiltin<"__builtin_ia32_ptestc128">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_sse41_ptestnzc : GCCBuiltin<"__builtin_ia32_ptestnzc128">, + def int_x86_sse41_ptestnzc : ClangBuiltin<"__builtin_ia32_ptestnzc128">, Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; } @@ -814,81 +824,81 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Miscellaneous // CRC Instruction let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse42_crc32_32_8 : GCCBuiltin<"__builtin_ia32_crc32qi">, + def int_x86_sse42_crc32_32_8 : ClangBuiltin<"__builtin_ia32_crc32qi">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_sse42_crc32_32_16 : GCCBuiltin<"__builtin_ia32_crc32hi">, + def int_x86_sse42_crc32_32_16 : ClangBuiltin<"__builtin_ia32_crc32hi">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_sse42_crc32_32_32 : GCCBuiltin<"__builtin_ia32_crc32si">, + def int_x86_sse42_crc32_32_32 : ClangBuiltin<"__builtin_ia32_crc32si">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_sse42_crc32_64_64 : GCCBuiltin<"__builtin_ia32_crc32di">, + def int_x86_sse42_crc32_64_64 : ClangBuiltin<"__builtin_ia32_crc32di">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; } // String/text processing ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse42_pcmpistrm128 : GCCBuiltin<"__builtin_ia32_pcmpistrm128">, + def int_x86_sse42_pcmpistrm128 : ClangBuiltin<"__builtin_ia32_pcmpistrm128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistri128 : GCCBuiltin<"__builtin_ia32_pcmpistri128">, + def int_x86_sse42_pcmpistri128 : ClangBuiltin<"__builtin_ia32_pcmpistri128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistria128 : GCCBuiltin<"__builtin_ia32_pcmpistria128">, + def int_x86_sse42_pcmpistria128 : ClangBuiltin<"__builtin_ia32_pcmpistria128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistric128 : GCCBuiltin<"__builtin_ia32_pcmpistric128">, + def int_x86_sse42_pcmpistric128 : ClangBuiltin<"__builtin_ia32_pcmpistric128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistrio128 : GCCBuiltin<"__builtin_ia32_pcmpistrio128">, + def int_x86_sse42_pcmpistrio128 : ClangBuiltin<"__builtin_ia32_pcmpistrio128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistris128 : GCCBuiltin<"__builtin_ia32_pcmpistris128">, + def int_x86_sse42_pcmpistris128 : ClangBuiltin<"__builtin_ia32_pcmpistris128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpistriz128 : GCCBuiltin<"__builtin_ia32_pcmpistriz128">, + def int_x86_sse42_pcmpistriz128 : ClangBuiltin<"__builtin_ia32_pcmpistriz128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sse42_pcmpestrm128 : GCCBuiltin<"__builtin_ia32_pcmpestrm128">, + def int_x86_sse42_pcmpestrm128 : ClangBuiltin<"__builtin_ia32_pcmpestrm128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestri128 : GCCBuiltin<"__builtin_ia32_pcmpestri128">, + def int_x86_sse42_pcmpestri128 : ClangBuiltin<"__builtin_ia32_pcmpestri128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestria128 : GCCBuiltin<"__builtin_ia32_pcmpestria128">, + def int_x86_sse42_pcmpestria128 : ClangBuiltin<"__builtin_ia32_pcmpestria128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestric128 : GCCBuiltin<"__builtin_ia32_pcmpestric128">, + def int_x86_sse42_pcmpestric128 : ClangBuiltin<"__builtin_ia32_pcmpestric128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestrio128 : GCCBuiltin<"__builtin_ia32_pcmpestrio128">, + def int_x86_sse42_pcmpestrio128 : ClangBuiltin<"__builtin_ia32_pcmpestrio128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestris128 : GCCBuiltin<"__builtin_ia32_pcmpestris128">, + def int_x86_sse42_pcmpestris128 : ClangBuiltin<"__builtin_ia32_pcmpestris128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_sse42_pcmpestriz128 : GCCBuiltin<"__builtin_ia32_pcmpestriz128">, + def int_x86_sse42_pcmpestriz128 : ClangBuiltin<"__builtin_ia32_pcmpestriz128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], @@ -899,17 +909,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // SSE4A let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse4a_extrqi : GCCBuiltin<"__builtin_ia32_extrqi">, + def int_x86_sse4a_extrqi : ClangBuiltin<"__builtin_ia32_extrqi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_sse4a_extrq : GCCBuiltin<"__builtin_ia32_extrq">, + def int_x86_sse4a_extrq : ClangBuiltin<"__builtin_ia32_extrq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_sse4a_insertqi : GCCBuiltin<"__builtin_ia32_insertqi">, + def int_x86_sse4a_insertqi : ClangBuiltin<"__builtin_ia32_insertqi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>; - def int_x86_sse4a_insertq : GCCBuiltin<"__builtin_ia32_insertq">, + def int_x86_sse4a_insertq : ClangBuiltin<"__builtin_ia32_insertq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; } @@ -918,177 +928,177 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_addsub_pd_256 : GCCBuiltin<"__builtin_ia32_addsubpd256">, + def int_x86_avx_addsub_pd_256 : ClangBuiltin<"__builtin_ia32_addsubpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_addsub_ps_256 : GCCBuiltin<"__builtin_ia32_addsubps256">, + def int_x86_avx_addsub_ps_256 : ClangBuiltin<"__builtin_ia32_addsubps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_max_pd_256 : GCCBuiltin<"__builtin_ia32_maxpd256">, + def int_x86_avx_max_pd_256 : ClangBuiltin<"__builtin_ia32_maxpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_max_ps_256 : GCCBuiltin<"__builtin_ia32_maxps256">, + def int_x86_avx_max_ps_256 : ClangBuiltin<"__builtin_ia32_maxps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_min_pd_256 : GCCBuiltin<"__builtin_ia32_minpd256">, + def int_x86_avx_min_pd_256 : ClangBuiltin<"__builtin_ia32_minpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_min_ps_256 : GCCBuiltin<"__builtin_ia32_minps256">, + def int_x86_avx_min_ps_256 : ClangBuiltin<"__builtin_ia32_minps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_rsqrt_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrtps256">, + def int_x86_avx_rsqrt_ps_256 : ClangBuiltin<"__builtin_ia32_rsqrtps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_rcp_ps_256 : GCCBuiltin<"__builtin_ia32_rcpps256">, + def int_x86_avx_rcp_ps_256 : ClangBuiltin<"__builtin_ia32_rcpps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_round_pd_256 : GCCBuiltin<"__builtin_ia32_roundpd256">, + def int_x86_avx_round_pd_256 : ClangBuiltin<"__builtin_ia32_roundpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx_round_ps_256 : GCCBuiltin<"__builtin_ia32_roundps256">, + def int_x86_avx_round_ps_256 : ClangBuiltin<"__builtin_ia32_roundps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } // Horizontal ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_hadd_pd_256 : GCCBuiltin<"__builtin_ia32_haddpd256">, + def int_x86_avx_hadd_pd_256 : ClangBuiltin<"__builtin_ia32_haddpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_hsub_ps_256 : GCCBuiltin<"__builtin_ia32_hsubps256">, + def int_x86_avx_hsub_ps_256 : ClangBuiltin<"__builtin_ia32_hsubps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_hsub_pd_256 : GCCBuiltin<"__builtin_ia32_hsubpd256">, + def int_x86_avx_hsub_pd_256 : ClangBuiltin<"__builtin_ia32_hsubpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_hadd_ps_256 : GCCBuiltin<"__builtin_ia32_haddps256">, + def int_x86_avx_hadd_ps_256 : ClangBuiltin<"__builtin_ia32_haddps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; } // Vector permutation let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_vpermilvar_pd : GCCBuiltin<"__builtin_ia32_vpermilvarpd">, + def int_x86_avx_vpermilvar_pd : ClangBuiltin<"__builtin_ia32_vpermilvarpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx_vpermilvar_ps : GCCBuiltin<"__builtin_ia32_vpermilvarps">, + def int_x86_avx_vpermilvar_ps : ClangBuiltin<"__builtin_ia32_vpermilvarps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx_vpermilvar_pd_256 : - GCCBuiltin<"__builtin_ia32_vpermilvarpd256">, + ClangBuiltin<"__builtin_ia32_vpermilvarpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx_vpermilvar_ps_256 : - GCCBuiltin<"__builtin_ia32_vpermilvarps256">, + ClangBuiltin<"__builtin_ia32_vpermilvarps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_d_128 : - GCCBuiltin<"__builtin_ia32_vpermi2vard128">, + ClangBuiltin<"__builtin_ia32_vpermi2vard128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_d_256 : - GCCBuiltin<"__builtin_ia32_vpermi2vard256">, + ClangBuiltin<"__builtin_ia32_vpermi2vard256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_d_512 : - GCCBuiltin<"__builtin_ia32_vpermi2vard512">, + ClangBuiltin<"__builtin_ia32_vpermi2vard512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_hi_128 : - GCCBuiltin<"__builtin_ia32_vpermi2varhi128">, + ClangBuiltin<"__builtin_ia32_vpermi2varhi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_hi_256 : - GCCBuiltin<"__builtin_ia32_vpermi2varhi256">, + ClangBuiltin<"__builtin_ia32_vpermi2varhi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_hi_512 : - GCCBuiltin<"__builtin_ia32_vpermi2varhi512">, + ClangBuiltin<"__builtin_ia32_vpermi2varhi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_pd_128 : - GCCBuiltin<"__builtin_ia32_vpermi2varpd128">, + ClangBuiltin<"__builtin_ia32_vpermi2varpd128">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_v2f64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_pd_256 : - GCCBuiltin<"__builtin_ia32_vpermi2varpd256">, + ClangBuiltin<"__builtin_ia32_vpermi2varpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_v4f64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_pd_512 : - GCCBuiltin<"__builtin_ia32_vpermi2varpd512">, + ClangBuiltin<"__builtin_ia32_vpermi2varpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_v8f64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_ps_128 : - GCCBuiltin<"__builtin_ia32_vpermi2varps128">, + ClangBuiltin<"__builtin_ia32_vpermi2varps128">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_ps_256 : - GCCBuiltin<"__builtin_ia32_vpermi2varps256">, + ClangBuiltin<"__builtin_ia32_vpermi2varps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8f32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_ps_512 : - GCCBuiltin<"__builtin_ia32_vpermi2varps512">, + ClangBuiltin<"__builtin_ia32_vpermi2varps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16f32_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_q_128 : - GCCBuiltin<"__builtin_ia32_vpermi2varq128">, + ClangBuiltin<"__builtin_ia32_vpermi2varq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_q_256 : - GCCBuiltin<"__builtin_ia32_vpermi2varq256">, + ClangBuiltin<"__builtin_ia32_vpermi2varq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_q_512 : - GCCBuiltin<"__builtin_ia32_vpermi2varq512">, + ClangBuiltin<"__builtin_ia32_vpermi2varq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_qi_128 : - GCCBuiltin<"__builtin_ia32_vpermi2varqi128">, + ClangBuiltin<"__builtin_ia32_vpermi2varqi128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_qi_256 : - GCCBuiltin<"__builtin_ia32_vpermi2varqi256">, + ClangBuiltin<"__builtin_ia32_vpermi2varqi256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx512_vpermi2var_qi_512 : - GCCBuiltin<"__builtin_ia32_vpermi2varqi512">, + ClangBuiltin<"__builtin_ia32_vpermi2varqi512">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx512_vpermilvar_pd_512 : - GCCBuiltin<"__builtin_ia32_vpermilvarpd512">, + ClangBuiltin<"__builtin_ia32_vpermilvarpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8i64_ty], [IntrNoMem]>; def int_x86_avx512_vpermilvar_ps_512 : - GCCBuiltin<"__builtin_ia32_vpermilvarps512">, + ClangBuiltin<"__builtin_ia32_vpermilvarps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_pshuf_b_512 : - GCCBuiltin<"__builtin_ia32_pshufb512">, + ClangBuiltin<"__builtin_ia32_pshufb512">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; @@ -1097,49 +1107,49 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // GFNI Instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_vgf2p8affineinvqb_128 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v16qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v16qi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8affineinvqb_256 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v32qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v32qi">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8affineinvqb_512 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v64qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v64qi">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8affineqb_128 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v16qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineqb_v16qi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8affineqb_256 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v32qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineqb_v32qi">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8affineqb_512 : - GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v64qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8affineqb_v64qi">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_vgf2p8mulb_128 : - GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v16qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8mulb_v16qi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_vgf2p8mulb_256 : - GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v32qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8mulb_v32qi">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_vgf2p8mulb_512 : - GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v64qi">, + ClangBuiltin<"__builtin_ia32_vgf2p8mulb_v64qi">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; @@ -1147,17 +1157,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector blend let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_blendv_pd_256 : GCCBuiltin<"__builtin_ia32_blendvpd256">, + def int_x86_avx_blendv_pd_256 : ClangBuiltin<"__builtin_ia32_blendvpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_blendv_ps_256 : GCCBuiltin<"__builtin_ia32_blendvps256">, + def int_x86_avx_blendv_ps_256 : ClangBuiltin<"__builtin_ia32_blendvps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; } // Vector dot product let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_dp_ps_256 : GCCBuiltin<"__builtin_ia32_dpps256">, + def int_x86_avx_dp_ps_256 : ClangBuiltin<"__builtin_ia32_dpps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, Commutative, ImmArg<ArgIndex<2>>]>; @@ -1175,63 +1185,63 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector convert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_cvt_pd2_ps_256 : GCCBuiltin<"__builtin_ia32_cvtpd2ps256">, + def int_x86_avx_cvt_pd2_ps_256 : ClangBuiltin<"__builtin_ia32_cvtpd2ps256">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_cvt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvtps2dq256">, + def int_x86_avx_cvt_ps2dq_256 : ClangBuiltin<"__builtin_ia32_cvtps2dq256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_cvtt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2dq256">, + def int_x86_avx_cvtt_pd2dq_256 : ClangBuiltin<"__builtin_ia32_cvttpd2dq256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_cvt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2dq256">, + def int_x86_avx_cvt_pd2dq_256 : ClangBuiltin<"__builtin_ia32_cvtpd2dq256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_cvtt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvttps2dq256">, + def int_x86_avx_cvtt_ps2dq_256 : ClangBuiltin<"__builtin_ia32_cvttps2dq256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>; } // Vector bit test let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_vtestz_pd : GCCBuiltin<"__builtin_ia32_vtestzpd">, + def int_x86_avx_vtestz_pd : ClangBuiltin<"__builtin_ia32_vtestzpd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestc_pd : GCCBuiltin<"__builtin_ia32_vtestcpd">, + def int_x86_avx_vtestc_pd : ClangBuiltin<"__builtin_ia32_vtestcpd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestnzc_pd : GCCBuiltin<"__builtin_ia32_vtestnzcpd">, + def int_x86_avx_vtestnzc_pd : ClangBuiltin<"__builtin_ia32_vtestnzcpd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestz_ps : GCCBuiltin<"__builtin_ia32_vtestzps">, + def int_x86_avx_vtestz_ps : ClangBuiltin<"__builtin_ia32_vtestzps">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx_vtestc_ps : GCCBuiltin<"__builtin_ia32_vtestcps">, + def int_x86_avx_vtestc_ps : ClangBuiltin<"__builtin_ia32_vtestcps">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx_vtestnzc_ps : GCCBuiltin<"__builtin_ia32_vtestnzcps">, + def int_x86_avx_vtestnzc_ps : ClangBuiltin<"__builtin_ia32_vtestnzcps">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx_vtestz_pd_256 : GCCBuiltin<"__builtin_ia32_vtestzpd256">, + def int_x86_avx_vtestz_pd_256 : ClangBuiltin<"__builtin_ia32_vtestzpd256">, Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestc_pd_256 : GCCBuiltin<"__builtin_ia32_vtestcpd256">, + def int_x86_avx_vtestc_pd_256 : ClangBuiltin<"__builtin_ia32_vtestcpd256">, Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestnzc_pd_256 : GCCBuiltin<"__builtin_ia32_vtestnzcpd256">, + def int_x86_avx_vtestnzc_pd_256 : ClangBuiltin<"__builtin_ia32_vtestnzcpd256">, Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_vtestz_ps_256 : GCCBuiltin<"__builtin_ia32_vtestzps256">, + def int_x86_avx_vtestz_ps_256 : ClangBuiltin<"__builtin_ia32_vtestzps256">, Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_vtestc_ps_256 : GCCBuiltin<"__builtin_ia32_vtestcps256">, + def int_x86_avx_vtestc_ps_256 : ClangBuiltin<"__builtin_ia32_vtestcps256">, Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_vtestnzc_ps_256 : GCCBuiltin<"__builtin_ia32_vtestnzcps256">, + def int_x86_avx_vtestnzc_ps_256 : ClangBuiltin<"__builtin_ia32_vtestnzcps256">, Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_ptestz_256 : GCCBuiltin<"__builtin_ia32_ptestz256">, + def int_x86_avx_ptestz_256 : ClangBuiltin<"__builtin_ia32_ptestz256">, Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx_ptestc_256 : GCCBuiltin<"__builtin_ia32_ptestc256">, + def int_x86_avx_ptestc_256 : ClangBuiltin<"__builtin_ia32_ptestc256">, Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx_ptestnzc_256 : GCCBuiltin<"__builtin_ia32_ptestnzc256">, + def int_x86_avx_ptestnzc_256 : ClangBuiltin<"__builtin_ia32_ptestnzc256">, Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; @@ -1254,67 +1264,67 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16i1_ty], [llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_fpclass_sd : - GCCBuiltin<"__builtin_ia32_fpclasssd_mask">, + ClangBuiltin<"__builtin_ia32_fpclasssd_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_fpclass_ss : - GCCBuiltin<"__builtin_ia32_fpclassss_mask">, + ClangBuiltin<"__builtin_ia32_fpclassss_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } // Vector extract sign mask let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_movmsk_pd_256 : GCCBuiltin<"__builtin_ia32_movmskpd256">, + def int_x86_avx_movmsk_pd_256 : ClangBuiltin<"__builtin_ia32_movmskpd256">, Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_movmsk_ps_256 : GCCBuiltin<"__builtin_ia32_movmskps256">, + def int_x86_avx_movmsk_ps_256 : ClangBuiltin<"__builtin_ia32_movmskps256">, Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty], [IntrNoMem]>; } // Vector zero let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_vzeroall : GCCBuiltin<"__builtin_ia32_vzeroall">, + def int_x86_avx_vzeroall : ClangBuiltin<"__builtin_ia32_vzeroall">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; - def int_x86_avx_vzeroupper : GCCBuiltin<"__builtin_ia32_vzeroupper">, + def int_x86_avx_vzeroupper : ClangBuiltin<"__builtin_ia32_vzeroupper">, Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; } // SIMD load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_ldu_dq_256 : GCCBuiltin<"__builtin_ia32_lddqu256">, + def int_x86_avx_ldu_dq_256 : ClangBuiltin<"__builtin_ia32_lddqu256">, Intrinsic<[llvm_v32i8_ty], [llvm_ptr_ty], [IntrReadMem]>; } // Conditional load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_maskload_pd : GCCBuiltin<"__builtin_ia32_maskloadpd">, + def int_x86_avx_maskload_pd : ClangBuiltin<"__builtin_ia32_maskloadpd">, Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2i64_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx_maskload_ps : GCCBuiltin<"__builtin_ia32_maskloadps">, + def int_x86_avx_maskload_ps : ClangBuiltin<"__builtin_ia32_maskloadps">, Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4i32_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx_maskload_pd_256 : GCCBuiltin<"__builtin_ia32_maskloadpd256">, + def int_x86_avx_maskload_pd_256 : ClangBuiltin<"__builtin_ia32_maskloadpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4i64_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx_maskload_ps_256 : GCCBuiltin<"__builtin_ia32_maskloadps256">, + def int_x86_avx_maskload_ps_256 : ClangBuiltin<"__builtin_ia32_maskloadps256">, Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8i32_ty], [IntrReadMem, IntrArgMemOnly]>; } // Conditional store ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_maskstore_pd : GCCBuiltin<"__builtin_ia32_maskstorepd">, + def int_x86_avx_maskstore_pd : ClangBuiltin<"__builtin_ia32_maskstorepd">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_v2f64_ty], [IntrArgMemOnly]>; - def int_x86_avx_maskstore_ps : GCCBuiltin<"__builtin_ia32_maskstoreps">, + def int_x86_avx_maskstore_ps : ClangBuiltin<"__builtin_ia32_maskstoreps">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f32_ty], [IntrArgMemOnly]>; def int_x86_avx_maskstore_pd_256 : - GCCBuiltin<"__builtin_ia32_maskstorepd256">, + ClangBuiltin<"__builtin_ia32_maskstorepd256">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f64_ty], [IntrArgMemOnly]>; def int_x86_avx_maskstore_ps_256 : - GCCBuiltin<"__builtin_ia32_maskstoreps256">, + ClangBuiltin<"__builtin_ia32_maskstoreps256">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_v8f32_ty], [IntrArgMemOnly]>; } @@ -1334,229 +1344,229 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Integer arithmetic ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw256">, + def int_x86_avx2_pmulhu_w : ClangBuiltin<"__builtin_ia32_pmulhuw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw256">, + def int_x86_avx2_pmulh_w : ClangBuiltin<"__builtin_ia32_pmulhw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd256">, + def int_x86_avx2_pmadd_wd : ClangBuiltin<"__builtin_ia32_pmaddwd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb256">, + def int_x86_avx2_pavg_b : ClangBuiltin<"__builtin_ia32_pavgb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw256">, + def int_x86_avx2_pavg_w : ClangBuiltin<"__builtin_ia32_pavgw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw256">, + def int_x86_avx2_psad_bw : ClangBuiltin<"__builtin_ia32_psadbw256">, Intrinsic<[llvm_v4i64_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem, Commutative]>; } // Integer shift ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_psll_w : GCCBuiltin<"__builtin_ia32_psllw256">, + def int_x86_avx2_psll_w : ClangBuiltin<"__builtin_ia32_psllw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx2_psll_d : GCCBuiltin<"__builtin_ia32_pslld256">, + def int_x86_avx2_psll_d : ClangBuiltin<"__builtin_ia32_pslld256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_psll_q : GCCBuiltin<"__builtin_ia32_psllq256">, + def int_x86_avx2_psll_q : ClangBuiltin<"__builtin_ia32_psllq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx2_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw256">, + def int_x86_avx2_psrl_w : ClangBuiltin<"__builtin_ia32_psrlw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx2_psrl_d : GCCBuiltin<"__builtin_ia32_psrld256">, + def int_x86_avx2_psrl_d : ClangBuiltin<"__builtin_ia32_psrld256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq256">, + def int_x86_avx2_psrl_q : ClangBuiltin<"__builtin_ia32_psrlq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx2_psra_w : GCCBuiltin<"__builtin_ia32_psraw256">, + def int_x86_avx2_psra_w : ClangBuiltin<"__builtin_ia32_psraw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx2_psra_d : GCCBuiltin<"__builtin_ia32_psrad256">, + def int_x86_avx2_psra_d : ClangBuiltin<"__builtin_ia32_psrad256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty], [IntrNoMem]>; // Oddly these don't require an immediate due to a gcc compatibility issue. - def int_x86_avx2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi256">, + def int_x86_avx2_pslli_w : ClangBuiltin<"__builtin_ia32_psllwi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi256">, + def int_x86_avx2_pslli_d : ClangBuiltin<"__builtin_ia32_pslldi256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi256">, + def int_x86_avx2_pslli_q : ClangBuiltin<"__builtin_ia32_psllqi256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi256">, + def int_x86_avx2_psrli_w : ClangBuiltin<"__builtin_ia32_psrlwi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi256">, + def int_x86_avx2_psrli_d : ClangBuiltin<"__builtin_ia32_psrldi256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi256">, + def int_x86_avx2_psrli_q : ClangBuiltin<"__builtin_ia32_psrlqi256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi256">, + def int_x86_avx2_psrai_w : ClangBuiltin<"__builtin_ia32_psrawi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrai_d : GCCBuiltin<"__builtin_ia32_psradi256">, + def int_x86_avx2_psrai_d : ClangBuiltin<"__builtin_ia32_psradi256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psra_q_128 : GCCBuiltin<"__builtin_ia32_psraq128">, + def int_x86_avx512_psra_q_128 : ClangBuiltin<"__builtin_ia32_psraq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx512_psra_q_256 : GCCBuiltin<"__builtin_ia32_psraq256">, + def int_x86_avx512_psra_q_256 : ClangBuiltin<"__builtin_ia32_psraq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty], [IntrNoMem]>; // Oddly these don't require an immediate due to a gcc compatibility issue. - def int_x86_avx512_psrai_q_128 : GCCBuiltin<"__builtin_ia32_psraqi128">, + def int_x86_avx512_psrai_q_128 : ClangBuiltin<"__builtin_ia32_psraqi128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrai_q_256 : GCCBuiltin<"__builtin_ia32_psraqi256">, + def int_x86_avx512_psrai_q_256 : ClangBuiltin<"__builtin_ia32_psraqi256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psll_w_512 : GCCBuiltin<"__builtin_ia32_psllw512">, + def int_x86_avx512_psll_w_512 : ClangBuiltin<"__builtin_ia32_psllw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psll_d_512 : GCCBuiltin<"__builtin_ia32_pslld512">, + def int_x86_avx512_psll_d_512 : ClangBuiltin<"__builtin_ia32_pslld512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx512_psll_q_512 : GCCBuiltin<"__builtin_ia32_psllq512">, + def int_x86_avx512_psll_q_512 : ClangBuiltin<"__builtin_ia32_psllq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx512_psrl_w_512 : GCCBuiltin<"__builtin_ia32_psrlw512">, + def int_x86_avx512_psrl_w_512 : ClangBuiltin<"__builtin_ia32_psrlw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrl_d_512 : GCCBuiltin<"__builtin_ia32_psrld512">, + def int_x86_avx512_psrl_d_512 : ClangBuiltin<"__builtin_ia32_psrld512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrl_q_512 : GCCBuiltin<"__builtin_ia32_psrlq512">, + def int_x86_avx512_psrl_q_512 : ClangBuiltin<"__builtin_ia32_psrlq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx512_psra_w_512 : GCCBuiltin<"__builtin_ia32_psraw512">, + def int_x86_avx512_psra_w_512 : ClangBuiltin<"__builtin_ia32_psraw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psra_d_512 : GCCBuiltin<"__builtin_ia32_psrad512">, + def int_x86_avx512_psra_d_512 : ClangBuiltin<"__builtin_ia32_psrad512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx512_psra_q_512 : GCCBuiltin<"__builtin_ia32_psraq512">, + def int_x86_avx512_psra_q_512 : ClangBuiltin<"__builtin_ia32_psraq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v2i64_ty], [IntrNoMem]>; // Oddly these don't require an immediate due to a gcc compatibility issue. - def int_x86_avx512_pslli_w_512 : GCCBuiltin<"__builtin_ia32_psllwi512">, + def int_x86_avx512_pslli_w_512 : ClangBuiltin<"__builtin_ia32_psllwi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_pslli_d_512 : GCCBuiltin<"__builtin_ia32_pslldi512">, + def int_x86_avx512_pslli_d_512 : ClangBuiltin<"__builtin_ia32_pslldi512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_pslli_q_512 : GCCBuiltin<"__builtin_ia32_psllqi512">, + def int_x86_avx512_pslli_q_512 : ClangBuiltin<"__builtin_ia32_psllqi512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrli_w_512 : GCCBuiltin<"__builtin_ia32_psrlwi512">, + def int_x86_avx512_psrli_w_512 : ClangBuiltin<"__builtin_ia32_psrlwi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrli_d_512 : GCCBuiltin<"__builtin_ia32_psrldi512">, + def int_x86_avx512_psrli_d_512 : ClangBuiltin<"__builtin_ia32_psrldi512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrli_q_512 : GCCBuiltin<"__builtin_ia32_psrlqi512">, + def int_x86_avx512_psrli_q_512 : ClangBuiltin<"__builtin_ia32_psrlqi512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrai_w_512 : GCCBuiltin<"__builtin_ia32_psrawi512">, + def int_x86_avx512_psrai_w_512 : ClangBuiltin<"__builtin_ia32_psrawi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrai_d_512 : GCCBuiltin<"__builtin_ia32_psradi512">, + def int_x86_avx512_psrai_d_512 : ClangBuiltin<"__builtin_ia32_psradi512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrai_q_512 : GCCBuiltin<"__builtin_ia32_psraqi512">, + def int_x86_avx512_psrai_q_512 : ClangBuiltin<"__builtin_ia32_psraqi512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_pmultishift_qb_128: - GCCBuiltin<"__builtin_ia32_vpmultishiftqb128">, + ClangBuiltin<"__builtin_ia32_vpmultishiftqb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx512_pmultishift_qb_256: - GCCBuiltin<"__builtin_ia32_vpmultishiftqb256">, + ClangBuiltin<"__builtin_ia32_vpmultishiftqb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx512_pmultishift_qb_512: - GCCBuiltin<"__builtin_ia32_vpmultishiftqb512">, + ClangBuiltin<"__builtin_ia32_vpmultishiftqb512">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; } // Pack ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_packsswb : GCCBuiltin<"__builtin_ia32_packsswb256">, + def int_x86_avx2_packsswb : ClangBuiltin<"__builtin_ia32_packsswb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_packssdw : GCCBuiltin<"__builtin_ia32_packssdw256">, + def int_x86_avx2_packssdw : ClangBuiltin<"__builtin_ia32_packssdw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_packuswb : GCCBuiltin<"__builtin_ia32_packuswb256">, + def int_x86_avx2_packuswb : ClangBuiltin<"__builtin_ia32_packuswb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_packusdw : GCCBuiltin<"__builtin_ia32_packusdw256">, + def int_x86_avx2_packusdw : ClangBuiltin<"__builtin_ia32_packusdw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; } // Horizontal arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_phadd_w : GCCBuiltin<"__builtin_ia32_phaddw256">, + def int_x86_avx2_phadd_w : ClangBuiltin<"__builtin_ia32_phaddw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_phadd_d : GCCBuiltin<"__builtin_ia32_phaddd256">, + def int_x86_avx2_phadd_d : ClangBuiltin<"__builtin_ia32_phaddd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_phadd_sw : GCCBuiltin<"__builtin_ia32_phaddsw256">, + def int_x86_avx2_phadd_sw : ClangBuiltin<"__builtin_ia32_phaddsw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_phsub_w : GCCBuiltin<"__builtin_ia32_phsubw256">, + def int_x86_avx2_phsub_w : ClangBuiltin<"__builtin_ia32_phsubw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_phsub_d : GCCBuiltin<"__builtin_ia32_phsubd256">, + def int_x86_avx2_phsub_d : ClangBuiltin<"__builtin_ia32_phsubd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_phsub_sw : GCCBuiltin<"__builtin_ia32_phsubsw256">, + def int_x86_avx2_phsub_sw : ClangBuiltin<"__builtin_ia32_phsubsw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_pmadd_ub_sw : GCCBuiltin<"__builtin_ia32_pmaddubsw256">, + def int_x86_avx2_pmadd_ub_sw : ClangBuiltin<"__builtin_ia32_pmaddubsw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; } // Sign ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_psign_b : GCCBuiltin<"__builtin_ia32_psignb256">, + def int_x86_avx2_psign_b : ClangBuiltin<"__builtin_ia32_psignb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; - def int_x86_avx2_psign_w : GCCBuiltin<"__builtin_ia32_psignw256">, + def int_x86_avx2_psign_w : ClangBuiltin<"__builtin_ia32_psignw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_psign_d : GCCBuiltin<"__builtin_ia32_psignd256">, + def int_x86_avx2_psign_d : ClangBuiltin<"__builtin_ia32_psignd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; } // Packed multiply high with round and scale let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pmul_hr_sw : GCCBuiltin<"__builtin_ia32_pmulhrsw256">, + def int_x86_avx2_pmul_hr_sw : ClangBuiltin<"__builtin_ia32_pmulhrsw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx512_pmul_hr_sw_512 : GCCBuiltin<"__builtin_ia32_pmulhrsw512">, + def int_x86_avx512_pmul_hr_sw_512 : ClangBuiltin<"__builtin_ia32_pmulhrsw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; } // Vector blend let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pblendvb : GCCBuiltin<"__builtin_ia32_pblendvb256">, + def int_x86_avx2_pblendvb : ClangBuiltin<"__builtin_ia32_pblendvb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; } @@ -1564,137 +1574,137 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector permutation let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_permd : GCCBuiltin<"__builtin_ia32_permvarsi256">, + def int_x86_avx2_permd : ClangBuiltin<"__builtin_ia32_permvarsi256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_permps : GCCBuiltin<"__builtin_ia32_permvarsf256">, + def int_x86_avx2_permps : ClangBuiltin<"__builtin_ia32_permvarsf256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>; } // Conditional load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_maskload_d : GCCBuiltin<"__builtin_ia32_maskloadd">, + def int_x86_avx2_maskload_d : ClangBuiltin<"__builtin_ia32_maskloadd">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_v4i32_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx2_maskload_q : GCCBuiltin<"__builtin_ia32_maskloadq">, + def int_x86_avx2_maskload_q : ClangBuiltin<"__builtin_ia32_maskloadq">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx2_maskload_d_256 : GCCBuiltin<"__builtin_ia32_maskloadd256">, + def int_x86_avx2_maskload_d_256 : ClangBuiltin<"__builtin_ia32_maskloadd256">, Intrinsic<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_v8i32_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx2_maskload_q_256 : GCCBuiltin<"__builtin_ia32_maskloadq256">, + def int_x86_avx2_maskload_q_256 : ClangBuiltin<"__builtin_ia32_maskloadq256">, Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_v4i64_ty], [IntrReadMem, IntrArgMemOnly]>; } // Conditional store ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_maskstore_d : GCCBuiltin<"__builtin_ia32_maskstored">, + def int_x86_avx2_maskstore_d : ClangBuiltin<"__builtin_ia32_maskstored">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrArgMemOnly]>; - def int_x86_avx2_maskstore_q : GCCBuiltin<"__builtin_ia32_maskstoreq">, + def int_x86_avx2_maskstore_q : ClangBuiltin<"__builtin_ia32_maskstoreq">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrArgMemOnly]>; def int_x86_avx2_maskstore_d_256 : - GCCBuiltin<"__builtin_ia32_maskstored256">, + ClangBuiltin<"__builtin_ia32_maskstored256">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrArgMemOnly]>; def int_x86_avx2_maskstore_q_256 : - GCCBuiltin<"__builtin_ia32_maskstoreq256">, + ClangBuiltin<"__builtin_ia32_maskstoreq256">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty], [IntrArgMemOnly]>; } // Variable bit shift ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_psllv_d : GCCBuiltin<"__builtin_ia32_psllv4si">, + def int_x86_avx2_psllv_d : ClangBuiltin<"__builtin_ia32_psllv4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_psllv_d_256 : GCCBuiltin<"__builtin_ia32_psllv8si">, + def int_x86_avx2_psllv_d_256 : ClangBuiltin<"__builtin_ia32_psllv8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_psllv_q : GCCBuiltin<"__builtin_ia32_psllv2di">, + def int_x86_avx2_psllv_q : ClangBuiltin<"__builtin_ia32_psllv2di">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx2_psllv_q_256 : GCCBuiltin<"__builtin_ia32_psllv4di">, + def int_x86_avx2_psllv_q_256 : ClangBuiltin<"__builtin_ia32_psllv4di">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_psllv_d_512 : GCCBuiltin<"__builtin_ia32_psllv16si">, + def int_x86_avx512_psllv_d_512 : ClangBuiltin<"__builtin_ia32_psllv16si">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; - def int_x86_avx512_psllv_q_512 : GCCBuiltin<"__builtin_ia32_psllv8di">, + def int_x86_avx512_psllv_q_512 : ClangBuiltin<"__builtin_ia32_psllv8di">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_avx2_psrlv_d : GCCBuiltin<"__builtin_ia32_psrlv4si">, + def int_x86_avx2_psrlv_d : ClangBuiltin<"__builtin_ia32_psrlv4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrlv_d_256 : GCCBuiltin<"__builtin_ia32_psrlv8si">, + def int_x86_avx2_psrlv_d_256 : ClangBuiltin<"__builtin_ia32_psrlv8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrlv_q : GCCBuiltin<"__builtin_ia32_psrlv2di">, + def int_x86_avx2_psrlv_q : ClangBuiltin<"__builtin_ia32_psrlv2di">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx2_psrlv_q_256 : GCCBuiltin<"__builtin_ia32_psrlv4di">, + def int_x86_avx2_psrlv_q_256 : ClangBuiltin<"__builtin_ia32_psrlv4di">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_psrlv_d_512 : GCCBuiltin<"__builtin_ia32_psrlv16si">, + def int_x86_avx512_psrlv_d_512 : ClangBuiltin<"__builtin_ia32_psrlv16si">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrlv_q_512 : GCCBuiltin<"__builtin_ia32_psrlv8di">, + def int_x86_avx512_psrlv_q_512 : ClangBuiltin<"__builtin_ia32_psrlv8di">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_avx2_psrav_d : GCCBuiltin<"__builtin_ia32_psrav4si">, + def int_x86_avx2_psrav_d : ClangBuiltin<"__builtin_ia32_psrav4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_psrav_d_256 : GCCBuiltin<"__builtin_ia32_psrav8si">, + def int_x86_avx2_psrav_d_256 : ClangBuiltin<"__builtin_ia32_psrav8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_d_512 : GCCBuiltin<"__builtin_ia32_psrav16si">, + def int_x86_avx512_psrav_d_512 : ClangBuiltin<"__builtin_ia32_psrav16si">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_q_128 : GCCBuiltin<"__builtin_ia32_psravq128">, + def int_x86_avx512_psrav_q_128 : ClangBuiltin<"__builtin_ia32_psravq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_q_256 : GCCBuiltin<"__builtin_ia32_psravq256">, + def int_x86_avx512_psrav_q_256 : ClangBuiltin<"__builtin_ia32_psravq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_q_512 : GCCBuiltin<"__builtin_ia32_psrav8di">, + def int_x86_avx512_psrav_q_512 : ClangBuiltin<"__builtin_ia32_psrav8di">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_avx512_psllv_w_128 : GCCBuiltin<"__builtin_ia32_psllv8hi">, + def int_x86_avx512_psllv_w_128 : ClangBuiltin<"__builtin_ia32_psllv8hi">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psllv_w_256 : GCCBuiltin<"__builtin_ia32_psllv16hi">, + def int_x86_avx512_psllv_w_256 : ClangBuiltin<"__builtin_ia32_psllv16hi">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx512_psllv_w_512 : GCCBuiltin<"__builtin_ia32_psllv32hi">, + def int_x86_avx512_psllv_w_512 : ClangBuiltin<"__builtin_ia32_psllv32hi">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrlv_w_128 : GCCBuiltin<"__builtin_ia32_psrlv8hi">, + def int_x86_avx512_psrlv_w_128 : ClangBuiltin<"__builtin_ia32_psrlv8hi">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrlv_w_256 : GCCBuiltin<"__builtin_ia32_psrlv16hi">, + def int_x86_avx512_psrlv_w_256 : ClangBuiltin<"__builtin_ia32_psrlv16hi">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrlv_w_512 : GCCBuiltin<"__builtin_ia32_psrlv32hi">, + def int_x86_avx512_psrlv_w_512 : ClangBuiltin<"__builtin_ia32_psrlv32hi">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_w_128 : GCCBuiltin<"__builtin_ia32_psrav8hi">, + def int_x86_avx512_psrav_w_128 : ClangBuiltin<"__builtin_ia32_psrav8hi">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_w_256 : GCCBuiltin<"__builtin_ia32_psrav16hi">, + def int_x86_avx512_psrav_w_256 : ClangBuiltin<"__builtin_ia32_psrav16hi">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx512_psrav_w_512 : GCCBuiltin<"__builtin_ia32_psrav32hi">, + def int_x86_avx512_psrav_w_512 : ClangBuiltin<"__builtin_ia32_psrav32hi">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; } @@ -1703,68 +1713,68 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // NOTE: These can't be ArgMemOnly because you can put the address completely // in the index register. - def int_x86_avx2_gather_d_pd : GCCBuiltin<"__builtin_ia32_gatherd_pd">, + def int_x86_avx2_gather_d_pd : ClangBuiltin<"__builtin_ia32_gatherd_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_pd_256 : GCCBuiltin<"__builtin_ia32_gatherd_pd256">, + def int_x86_avx2_gather_d_pd_256 : ClangBuiltin<"__builtin_ia32_gatherd_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_pd : GCCBuiltin<"__builtin_ia32_gatherq_pd">, + def int_x86_avx2_gather_q_pd : ClangBuiltin<"__builtin_ia32_gatherq_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_pd_256 : GCCBuiltin<"__builtin_ia32_gatherq_pd256">, + def int_x86_avx2_gather_q_pd_256 : ClangBuiltin<"__builtin_ia32_gatherq_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_ps : GCCBuiltin<"__builtin_ia32_gatherd_ps">, + def int_x86_avx2_gather_d_ps : ClangBuiltin<"__builtin_ia32_gatherd_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_ps_256 : GCCBuiltin<"__builtin_ia32_gatherd_ps256">, + def int_x86_avx2_gather_d_ps_256 : ClangBuiltin<"__builtin_ia32_gatherd_ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_ps : GCCBuiltin<"__builtin_ia32_gatherq_ps">, + def int_x86_avx2_gather_q_ps : ClangBuiltin<"__builtin_ia32_gatherq_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_ps_256 : GCCBuiltin<"__builtin_ia32_gatherq_ps256">, + def int_x86_avx2_gather_q_ps_256 : ClangBuiltin<"__builtin_ia32_gatherq_ps256">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_q : GCCBuiltin<"__builtin_ia32_gatherd_q">, + def int_x86_avx2_gather_d_q : ClangBuiltin<"__builtin_ia32_gatherd_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_q_256 : GCCBuiltin<"__builtin_ia32_gatherd_q256">, + def int_x86_avx2_gather_d_q_256 : ClangBuiltin<"__builtin_ia32_gatherd_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_q : GCCBuiltin<"__builtin_ia32_gatherq_q">, + def int_x86_avx2_gather_q_q : ClangBuiltin<"__builtin_ia32_gatherq_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_q_256 : GCCBuiltin<"__builtin_ia32_gatherq_q256">, + def int_x86_avx2_gather_q_q_256 : ClangBuiltin<"__builtin_ia32_gatherq_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_d : GCCBuiltin<"__builtin_ia32_gatherd_d">, + def int_x86_avx2_gather_d_d : ClangBuiltin<"__builtin_ia32_gatherd_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_d_d_256 : GCCBuiltin<"__builtin_ia32_gatherd_d256">, + def int_x86_avx2_gather_d_d_256 : ClangBuiltin<"__builtin_ia32_gatherd_d256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_d : GCCBuiltin<"__builtin_ia32_gatherq_d">, + def int_x86_avx2_gather_q_d : ClangBuiltin<"__builtin_ia32_gatherq_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx2_gather_q_d_256 : GCCBuiltin<"__builtin_ia32_gatherq_d256">, + def int_x86_avx2_gather_q_d_256 : ClangBuiltin<"__builtin_ia32_gatherq_d256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrReadMem, ImmArg<ArgIndex<4>>]>; @@ -1772,12 +1782,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Misc. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pmovmskb : GCCBuiltin<"__builtin_ia32_pmovmskb256">, + def int_x86_avx2_pmovmskb : ClangBuiltin<"__builtin_ia32_pmovmskb256">, Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty], [IntrNoMem]>; - def int_x86_avx2_pshuf_b : GCCBuiltin<"__builtin_ia32_pshufb256">, + def int_x86_avx2_pshuf_b : ClangBuiltin<"__builtin_ia32_pshufb256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; - def int_x86_avx2_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw256">, + def int_x86_avx2_mpsadbw : ClangBuiltin<"__builtin_ia32_mpsadbw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } @@ -1786,21 +1796,21 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FMA3 and FMA4 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_fma_vfmaddsub_ps : GCCBuiltin<"__builtin_ia32_vfmaddsubps">, + def int_x86_fma_vfmaddsub_ps : ClangBuiltin<"__builtin_ia32_vfmaddsubps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmaddsub_pd : GCCBuiltin<"__builtin_ia32_vfmaddsubpd">, + def int_x86_fma_vfmaddsub_pd : ClangBuiltin<"__builtin_ia32_vfmaddsubpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; def int_x86_fma_vfmaddsub_ps_256 : - GCCBuiltin<"__builtin_ia32_vfmaddsubps256">, + ClangBuiltin<"__builtin_ia32_vfmaddsubps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; def int_x86_fma_vfmaddsub_pd_256 : - GCCBuiltin<"__builtin_ia32_vfmaddsubpd256">, + ClangBuiltin<"__builtin_ia32_vfmaddsubpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; @@ -1835,27 +1845,27 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_vpmadd52h_uq_128 : - GCCBuiltin<"__builtin_ia32_vpmadd52huq128">, + ClangBuiltin<"__builtin_ia32_vpmadd52huq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_vpmadd52l_uq_128 : - GCCBuiltin<"__builtin_ia32_vpmadd52luq128">, + ClangBuiltin<"__builtin_ia32_vpmadd52luq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_vpmadd52h_uq_256 : - GCCBuiltin<"__builtin_ia32_vpmadd52huq256">, + ClangBuiltin<"__builtin_ia32_vpmadd52huq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx512_vpmadd52l_uq_256 : - GCCBuiltin<"__builtin_ia32_vpmadd52luq256">, + ClangBuiltin<"__builtin_ia32_vpmadd52luq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx512_vpmadd52h_uq_512 : - GCCBuiltin<"__builtin_ia32_vpmadd52huq512">, + ClangBuiltin<"__builtin_ia32_vpmadd52huq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; def int_x86_avx512_vpmadd52l_uq_512 : - GCCBuiltin<"__builtin_ia32_vpmadd52luq512">, + ClangBuiltin<"__builtin_ia32_vpmadd52luq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; } @@ -1863,54 +1873,54 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // VNNI let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_vpdpbusd_128 : - GCCBuiltin<"__builtin_ia32_vpdpbusd128">, + ClangBuiltin<"__builtin_ia32_vpdpbusd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpbusd_256 : - GCCBuiltin<"__builtin_ia32_vpdpbusd256">, + ClangBuiltin<"__builtin_ia32_vpdpbusd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpbusd_512 : - GCCBuiltin<"__builtin_ia32_vpdpbusd512">, + ClangBuiltin<"__builtin_ia32_vpdpbusd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpbusds_128 : - GCCBuiltin<"__builtin_ia32_vpdpbusds128">, + ClangBuiltin<"__builtin_ia32_vpdpbusds128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpbusds_256 : - GCCBuiltin<"__builtin_ia32_vpdpbusds256">, + ClangBuiltin<"__builtin_ia32_vpdpbusds256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpbusds_512 : - GCCBuiltin<"__builtin_ia32_vpdpbusds512">, + ClangBuiltin<"__builtin_ia32_vpdpbusds512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssd_128 : - GCCBuiltin<"__builtin_ia32_vpdpwssd128">, + ClangBuiltin<"__builtin_ia32_vpdpwssd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssd_256 : - GCCBuiltin<"__builtin_ia32_vpdpwssd256">, + ClangBuiltin<"__builtin_ia32_vpdpwssd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssd_512 : - GCCBuiltin<"__builtin_ia32_vpdpwssd512">, + ClangBuiltin<"__builtin_ia32_vpdpwssd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssds_128 : - GCCBuiltin<"__builtin_ia32_vpdpwssds128">, + ClangBuiltin<"__builtin_ia32_vpdpwssds128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssds_256 : - GCCBuiltin<"__builtin_ia32_vpdpwssds256">, + ClangBuiltin<"__builtin_ia32_vpdpwssds256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_vpdpwssds_512 : - GCCBuiltin<"__builtin_ia32_vpdpwssds512">, + ClangBuiltin<"__builtin_ia32_vpdpwssds512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; } @@ -1919,180 +1929,180 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // XOP let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_xop_vpermil2pd : GCCBuiltin<"__builtin_ia32_vpermil2pd">, + def int_x86_xop_vpermil2pd : ClangBuiltin<"__builtin_ia32_vpermil2pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_xop_vpermil2pd_256 : - GCCBuiltin<"__builtin_ia32_vpermil2pd256">, + ClangBuiltin<"__builtin_ia32_vpermil2pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_xop_vpermil2ps : GCCBuiltin<"__builtin_ia32_vpermil2ps">, + def int_x86_xop_vpermil2ps : ClangBuiltin<"__builtin_ia32_vpermil2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_xop_vpermil2ps_256 : - GCCBuiltin<"__builtin_ia32_vpermil2ps256">, + ClangBuiltin<"__builtin_ia32_vpermil2ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_xop_vfrcz_pd : GCCBuiltin<"__builtin_ia32_vfrczpd">, + def int_x86_xop_vfrcz_pd : ClangBuiltin<"__builtin_ia32_vfrczpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_xop_vfrcz_ps : GCCBuiltin<"__builtin_ia32_vfrczps">, + def int_x86_xop_vfrcz_ps : ClangBuiltin<"__builtin_ia32_vfrczps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_xop_vfrcz_sd : GCCBuiltin<"__builtin_ia32_vfrczsd">, + def int_x86_xop_vfrcz_sd : ClangBuiltin<"__builtin_ia32_vfrczsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_xop_vfrcz_ss : GCCBuiltin<"__builtin_ia32_vfrczss">, + def int_x86_xop_vfrcz_ss : ClangBuiltin<"__builtin_ia32_vfrczss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_xop_vfrcz_pd_256 : GCCBuiltin<"__builtin_ia32_vfrczpd256">, + def int_x86_xop_vfrcz_pd_256 : ClangBuiltin<"__builtin_ia32_vfrczpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_xop_vfrcz_ps_256 : GCCBuiltin<"__builtin_ia32_vfrczps256">, + def int_x86_xop_vfrcz_ps_256 : ClangBuiltin<"__builtin_ia32_vfrczps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>; def int_x86_xop_vphaddbd : - GCCBuiltin<"__builtin_ia32_vphaddbd">, + ClangBuiltin<"__builtin_ia32_vphaddbd">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphaddbq : - GCCBuiltin<"__builtin_ia32_vphaddbq">, + ClangBuiltin<"__builtin_ia32_vphaddbq">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphaddbw : - GCCBuiltin<"__builtin_ia32_vphaddbw">, + ClangBuiltin<"__builtin_ia32_vphaddbw">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphadddq : - GCCBuiltin<"__builtin_ia32_vphadddq">, + ClangBuiltin<"__builtin_ia32_vphadddq">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_xop_vphaddubd : - GCCBuiltin<"__builtin_ia32_vphaddubd">, + ClangBuiltin<"__builtin_ia32_vphaddubd">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphaddubq : - GCCBuiltin<"__builtin_ia32_vphaddubq">, + ClangBuiltin<"__builtin_ia32_vphaddubq">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphaddubw : - GCCBuiltin<"__builtin_ia32_vphaddubw">, + ClangBuiltin<"__builtin_ia32_vphaddubw">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphaddudq : - GCCBuiltin<"__builtin_ia32_vphaddudq">, + ClangBuiltin<"__builtin_ia32_vphaddudq">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_xop_vphadduwd : - GCCBuiltin<"__builtin_ia32_vphadduwd">, + ClangBuiltin<"__builtin_ia32_vphadduwd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vphadduwq : - GCCBuiltin<"__builtin_ia32_vphadduwq">, + ClangBuiltin<"__builtin_ia32_vphadduwq">, Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vphaddwd : - GCCBuiltin<"__builtin_ia32_vphaddwd">, + ClangBuiltin<"__builtin_ia32_vphaddwd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vphaddwq : - GCCBuiltin<"__builtin_ia32_vphaddwq">, + ClangBuiltin<"__builtin_ia32_vphaddwq">, Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vphsubbw : - GCCBuiltin<"__builtin_ia32_vphsubbw">, + ClangBuiltin<"__builtin_ia32_vphsubbw">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vphsubdq : - GCCBuiltin<"__builtin_ia32_vphsubdq">, + ClangBuiltin<"__builtin_ia32_vphsubdq">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_xop_vphsubwd : - GCCBuiltin<"__builtin_ia32_vphsubwd">, + ClangBuiltin<"__builtin_ia32_vphsubwd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vpmacsdd : - GCCBuiltin<"__builtin_ia32_vpmacsdd">, + ClangBuiltin<"__builtin_ia32_vpmacsdd">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacsdqh : - GCCBuiltin<"__builtin_ia32_vpmacsdqh">, + ClangBuiltin<"__builtin_ia32_vpmacsdqh">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacsdql : - GCCBuiltin<"__builtin_ia32_vpmacsdql">, + ClangBuiltin<"__builtin_ia32_vpmacsdql">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacssdd : - GCCBuiltin<"__builtin_ia32_vpmacssdd">, + ClangBuiltin<"__builtin_ia32_vpmacssdd">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacssdqh : - GCCBuiltin<"__builtin_ia32_vpmacssdqh">, + ClangBuiltin<"__builtin_ia32_vpmacssdqh">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacssdql : - GCCBuiltin<"__builtin_ia32_vpmacssdql">, + ClangBuiltin<"__builtin_ia32_vpmacssdql">, Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacsswd : - GCCBuiltin<"__builtin_ia32_vpmacsswd">, + ClangBuiltin<"__builtin_ia32_vpmacsswd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacssww : - GCCBuiltin<"__builtin_ia32_vpmacssww">, + ClangBuiltin<"__builtin_ia32_vpmacssww">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacswd : - GCCBuiltin<"__builtin_ia32_vpmacswd">, + ClangBuiltin<"__builtin_ia32_vpmacswd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmacsww : - GCCBuiltin<"__builtin_ia32_vpmacsww">, + ClangBuiltin<"__builtin_ia32_vpmacsww">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmadcsswd : - GCCBuiltin<"__builtin_ia32_vpmadcsswd">, + ClangBuiltin<"__builtin_ia32_vpmadcsswd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpmadcswd : - GCCBuiltin<"__builtin_ia32_vpmadcswd">, + ClangBuiltin<"__builtin_ia32_vpmadcswd">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty], [IntrNoMem, Commutative]>; def int_x86_xop_vpperm : - GCCBuiltin<"__builtin_ia32_vpperm">, + ClangBuiltin<"__builtin_ia32_vpperm">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vpshab : - GCCBuiltin<"__builtin_ia32_vpshab">, + ClangBuiltin<"__builtin_ia32_vpshab">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vpshad : - GCCBuiltin<"__builtin_ia32_vpshad">, + ClangBuiltin<"__builtin_ia32_vpshad">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_xop_vpshaq : - GCCBuiltin<"__builtin_ia32_vpshaq">, + ClangBuiltin<"__builtin_ia32_vpshaq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_xop_vpshaw : - GCCBuiltin<"__builtin_ia32_vpshaw">, + ClangBuiltin<"__builtin_ia32_vpshaw">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; def int_x86_xop_vpshlb : - GCCBuiltin<"__builtin_ia32_vpshlb">, + ClangBuiltin<"__builtin_ia32_vpshlb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_xop_vpshld : - GCCBuiltin<"__builtin_ia32_vpshld">, + ClangBuiltin<"__builtin_ia32_vpshld">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_xop_vpshlq : - GCCBuiltin<"__builtin_ia32_vpshlq">, + ClangBuiltin<"__builtin_ia32_vpshlq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_x86_xop_vpshlw : - GCCBuiltin<"__builtin_ia32_vpshlw">, + ClangBuiltin<"__builtin_ia32_vpshlw">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; } @@ -2101,25 +2111,25 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // LWP let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_llwpcb : - GCCBuiltin<"__builtin_ia32_llwpcb">, + ClangBuiltin<"__builtin_ia32_llwpcb">, Intrinsic<[], [llvm_ptr_ty], []>; def int_x86_slwpcb : - GCCBuiltin<"__builtin_ia32_slwpcb">, + ClangBuiltin<"__builtin_ia32_slwpcb">, Intrinsic<[llvm_ptr_ty], [], []>; def int_x86_lwpins32 : - GCCBuiltin<"__builtin_ia32_lwpins32">, + ClangBuiltin<"__builtin_ia32_lwpins32">, Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; def int_x86_lwpins64 : - GCCBuiltin<"__builtin_ia32_lwpins64">, + ClangBuiltin<"__builtin_ia32_lwpins64">, Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; def int_x86_lwpval32 : - GCCBuiltin<"__builtin_ia32_lwpval32">, + ClangBuiltin<"__builtin_ia32_lwpval32">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; def int_x86_lwpval64 : - GCCBuiltin<"__builtin_ia32_lwpval64">, + ClangBuiltin<"__builtin_ia32_lwpval64">, Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; } @@ -2129,127 +2139,127 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Empty MMX state op. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_mmx_emms : GCCBuiltin<"__builtin_ia32_emms">, + def int_x86_mmx_emms : ClangBuiltin<"__builtin_ia32_emms">, Intrinsic<[], [], []>; - def int_x86_mmx_femms : GCCBuiltin<"__builtin_ia32_femms">, + def int_x86_mmx_femms : ClangBuiltin<"__builtin_ia32_femms">, Intrinsic<[], [], []>; } // Integer arithmetic ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Addition - def int_x86_mmx_padd_b : GCCBuiltin<"__builtin_ia32_paddb">, + def int_x86_mmx_padd_b : ClangBuiltin<"__builtin_ia32_paddb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_padd_w : GCCBuiltin<"__builtin_ia32_paddw">, + def int_x86_mmx_padd_w : ClangBuiltin<"__builtin_ia32_paddw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_padd_d : GCCBuiltin<"__builtin_ia32_paddd">, + def int_x86_mmx_padd_d : ClangBuiltin<"__builtin_ia32_paddd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_padd_q : GCCBuiltin<"__builtin_ia32_paddq">, + def int_x86_mmx_padd_q : ClangBuiltin<"__builtin_ia32_paddq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_padds_b : GCCBuiltin<"__builtin_ia32_paddsb">, + def int_x86_mmx_padds_b : ClangBuiltin<"__builtin_ia32_paddsb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_padds_w : GCCBuiltin<"__builtin_ia32_paddsw">, + def int_x86_mmx_padds_w : ClangBuiltin<"__builtin_ia32_paddsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_paddus_b : GCCBuiltin<"__builtin_ia32_paddusb">, + def int_x86_mmx_paddus_b : ClangBuiltin<"__builtin_ia32_paddusb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_paddus_w : GCCBuiltin<"__builtin_ia32_paddusw">, + def int_x86_mmx_paddus_w : ClangBuiltin<"__builtin_ia32_paddusw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Subtraction - def int_x86_mmx_psub_b : GCCBuiltin<"__builtin_ia32_psubb">, + def int_x86_mmx_psub_b : ClangBuiltin<"__builtin_ia32_psubb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psub_w : GCCBuiltin<"__builtin_ia32_psubw">, + def int_x86_mmx_psub_w : ClangBuiltin<"__builtin_ia32_psubw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psub_d : GCCBuiltin<"__builtin_ia32_psubd">, + def int_x86_mmx_psub_d : ClangBuiltin<"__builtin_ia32_psubd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psub_q : GCCBuiltin<"__builtin_ia32_psubq">, + def int_x86_mmx_psub_q : ClangBuiltin<"__builtin_ia32_psubq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psubs_b : GCCBuiltin<"__builtin_ia32_psubsb">, + def int_x86_mmx_psubs_b : ClangBuiltin<"__builtin_ia32_psubsb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psubs_w : GCCBuiltin<"__builtin_ia32_psubsw">, + def int_x86_mmx_psubs_w : ClangBuiltin<"__builtin_ia32_psubsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psubus_b : GCCBuiltin<"__builtin_ia32_psubusb">, + def int_x86_mmx_psubus_b : ClangBuiltin<"__builtin_ia32_psubusb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psubus_w : GCCBuiltin<"__builtin_ia32_psubusw">, + def int_x86_mmx_psubus_w : ClangBuiltin<"__builtin_ia32_psubusw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; // Multiplication - def int_x86_mmx_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw">, + def int_x86_mmx_pmulh_w : ClangBuiltin<"__builtin_ia32_pmulhw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmull_w : GCCBuiltin<"__builtin_ia32_pmullw">, + def int_x86_mmx_pmull_w : ClangBuiltin<"__builtin_ia32_pmullw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw">, + def int_x86_mmx_pmulhu_w : ClangBuiltin<"__builtin_ia32_pmulhuw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmulu_dq : GCCBuiltin<"__builtin_ia32_pmuludq">, + def int_x86_mmx_pmulu_dq : ClangBuiltin<"__builtin_ia32_pmuludq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd">, + def int_x86_mmx_pmadd_wd : ClangBuiltin<"__builtin_ia32_pmaddwd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Bitwise operations - def int_x86_mmx_pand : GCCBuiltin<"__builtin_ia32_pand">, + def int_x86_mmx_pand : ClangBuiltin<"__builtin_ia32_pand">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pandn : GCCBuiltin<"__builtin_ia32_pandn">, + def int_x86_mmx_pandn : ClangBuiltin<"__builtin_ia32_pandn">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_por : GCCBuiltin<"__builtin_ia32_por">, + def int_x86_mmx_por : ClangBuiltin<"__builtin_ia32_por">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pxor : GCCBuiltin<"__builtin_ia32_pxor">, + def int_x86_mmx_pxor : ClangBuiltin<"__builtin_ia32_pxor">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Averages - def int_x86_mmx_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb">, + def int_x86_mmx_pavg_b : ClangBuiltin<"__builtin_ia32_pavgb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw">, + def int_x86_mmx_pavg_w : ClangBuiltin<"__builtin_ia32_pavgw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Maximum - def int_x86_mmx_pmaxu_b : GCCBuiltin<"__builtin_ia32_pmaxub">, + def int_x86_mmx_pmaxu_b : ClangBuiltin<"__builtin_ia32_pmaxub">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmaxs_w : GCCBuiltin<"__builtin_ia32_pmaxsw">, + def int_x86_mmx_pmaxs_w : ClangBuiltin<"__builtin_ia32_pmaxsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Minimum - def int_x86_mmx_pminu_b : GCCBuiltin<"__builtin_ia32_pminub">, + def int_x86_mmx_pminu_b : ClangBuiltin<"__builtin_ia32_pminub">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pmins_w : GCCBuiltin<"__builtin_ia32_pminsw">, + def int_x86_mmx_pmins_w : ClangBuiltin<"__builtin_ia32_pminsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; // Packed sum of absolute differences - def int_x86_mmx_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw">, + def int_x86_mmx_psad_bw : ClangBuiltin<"__builtin_ia32_psadbw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; } @@ -2257,178 +2267,178 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Integer shift ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Shift left logical - def int_x86_mmx_psll_w : GCCBuiltin<"__builtin_ia32_psllw">, + def int_x86_mmx_psll_w : ClangBuiltin<"__builtin_ia32_psllw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psll_d : GCCBuiltin<"__builtin_ia32_pslld">, + def int_x86_mmx_psll_d : ClangBuiltin<"__builtin_ia32_pslld">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psll_q : GCCBuiltin<"__builtin_ia32_psllq">, + def int_x86_mmx_psll_q : ClangBuiltin<"__builtin_ia32_psllq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw">, + def int_x86_mmx_psrl_w : ClangBuiltin<"__builtin_ia32_psrlw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psrl_d : GCCBuiltin<"__builtin_ia32_psrld">, + def int_x86_mmx_psrl_d : ClangBuiltin<"__builtin_ia32_psrld">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq">, + def int_x86_mmx_psrl_q : ClangBuiltin<"__builtin_ia32_psrlq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psra_w : GCCBuiltin<"__builtin_ia32_psraw">, + def int_x86_mmx_psra_w : ClangBuiltin<"__builtin_ia32_psraw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_psra_d : GCCBuiltin<"__builtin_ia32_psrad">, + def int_x86_mmx_psra_d : ClangBuiltin<"__builtin_ia32_psrad">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; // Oddly these don't require an immediate due to a gcc compatibility issue. - def int_x86_mmx_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi">, + def int_x86_mmx_pslli_w : ClangBuiltin<"__builtin_ia32_psllwi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi">, + def int_x86_mmx_pslli_d : ClangBuiltin<"__builtin_ia32_pslldi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi">, + def int_x86_mmx_pslli_q : ClangBuiltin<"__builtin_ia32_psllqi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi">, + def int_x86_mmx_psrli_w : ClangBuiltin<"__builtin_ia32_psrlwi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi">, + def int_x86_mmx_psrli_d : ClangBuiltin<"__builtin_ia32_psrldi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi">, + def int_x86_mmx_psrli_q : ClangBuiltin<"__builtin_ia32_psrlqi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi">, + def int_x86_mmx_psrai_w : ClangBuiltin<"__builtin_ia32_psrawi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_mmx_psrai_d : GCCBuiltin<"__builtin_ia32_psradi">, + def int_x86_mmx_psrai_d : ClangBuiltin<"__builtin_ia32_psradi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; } // Permute let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_permvar_df_256 : GCCBuiltin<"__builtin_ia32_permvardf256">, + def int_x86_avx512_permvar_df_256 : ClangBuiltin<"__builtin_ia32_permvardf256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_df_512 : GCCBuiltin<"__builtin_ia32_permvardf512">, + def int_x86_avx512_permvar_df_512 : ClangBuiltin<"__builtin_ia32_permvardf512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_di_256 : GCCBuiltin<"__builtin_ia32_permvardi256">, + def int_x86_avx512_permvar_di_256 : ClangBuiltin<"__builtin_ia32_permvardi256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_di_512 : GCCBuiltin<"__builtin_ia32_permvardi512">, + def int_x86_avx512_permvar_di_512 : ClangBuiltin<"__builtin_ia32_permvardi512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_hi_128 : GCCBuiltin<"__builtin_ia32_permvarhi128">, + def int_x86_avx512_permvar_hi_128 : ClangBuiltin<"__builtin_ia32_permvarhi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_hi_256 : GCCBuiltin<"__builtin_ia32_permvarhi256">, + def int_x86_avx512_permvar_hi_256 : ClangBuiltin<"__builtin_ia32_permvarhi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_hi_512 : GCCBuiltin<"__builtin_ia32_permvarhi512">, + def int_x86_avx512_permvar_hi_512 : ClangBuiltin<"__builtin_ia32_permvarhi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_qi_128 : GCCBuiltin<"__builtin_ia32_permvarqi128">, + def int_x86_avx512_permvar_qi_128 : ClangBuiltin<"__builtin_ia32_permvarqi128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_qi_256 : GCCBuiltin<"__builtin_ia32_permvarqi256">, + def int_x86_avx512_permvar_qi_256 : ClangBuiltin<"__builtin_ia32_permvarqi256">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_qi_512 : GCCBuiltin<"__builtin_ia32_permvarqi512">, + def int_x86_avx512_permvar_qi_512 : ClangBuiltin<"__builtin_ia32_permvarqi512">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_sf_512 : GCCBuiltin<"__builtin_ia32_permvarsf512">, + def int_x86_avx512_permvar_sf_512 : ClangBuiltin<"__builtin_ia32_permvarsf512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty], [IntrNoMem]>; - def int_x86_avx512_permvar_si_512 : GCCBuiltin<"__builtin_ia32_permvarsi512">, + def int_x86_avx512_permvar_si_512 : ClangBuiltin<"__builtin_ia32_permvarsi512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; } // Pack ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_mmx_packsswb : GCCBuiltin<"__builtin_ia32_packsswb">, + def int_x86_mmx_packsswb : ClangBuiltin<"__builtin_ia32_packsswb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_packssdw : GCCBuiltin<"__builtin_ia32_packssdw">, + def int_x86_mmx_packssdw : ClangBuiltin<"__builtin_ia32_packssdw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_packuswb : GCCBuiltin<"__builtin_ia32_packuswb">, + def int_x86_mmx_packuswb : ClangBuiltin<"__builtin_ia32_packuswb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; } // Unpacking ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_mmx_punpckhbw : GCCBuiltin<"__builtin_ia32_punpckhbw">, + def int_x86_mmx_punpckhbw : ClangBuiltin<"__builtin_ia32_punpckhbw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_punpckhwd : GCCBuiltin<"__builtin_ia32_punpckhwd">, + def int_x86_mmx_punpckhwd : ClangBuiltin<"__builtin_ia32_punpckhwd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_punpckhdq : GCCBuiltin<"__builtin_ia32_punpckhdq">, + def int_x86_mmx_punpckhdq : ClangBuiltin<"__builtin_ia32_punpckhdq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_punpcklbw : GCCBuiltin<"__builtin_ia32_punpcklbw">, + def int_x86_mmx_punpcklbw : ClangBuiltin<"__builtin_ia32_punpcklbw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_punpcklwd : GCCBuiltin<"__builtin_ia32_punpcklwd">, + def int_x86_mmx_punpcklwd : ClangBuiltin<"__builtin_ia32_punpcklwd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_punpckldq : GCCBuiltin<"__builtin_ia32_punpckldq">, + def int_x86_mmx_punpckldq : ClangBuiltin<"__builtin_ia32_punpckldq">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; } // Integer comparison ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_mmx_pcmpeq_b : GCCBuiltin<"__builtin_ia32_pcmpeqb">, + def int_x86_mmx_pcmpeq_b : ClangBuiltin<"__builtin_ia32_pcmpeqb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pcmpeq_w : GCCBuiltin<"__builtin_ia32_pcmpeqw">, + def int_x86_mmx_pcmpeq_w : ClangBuiltin<"__builtin_ia32_pcmpeqw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pcmpeq_d : GCCBuiltin<"__builtin_ia32_pcmpeqd">, + def int_x86_mmx_pcmpeq_d : ClangBuiltin<"__builtin_ia32_pcmpeqd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem, Commutative]>; - def int_x86_mmx_pcmpgt_b : GCCBuiltin<"__builtin_ia32_pcmpgtb">, + def int_x86_mmx_pcmpgt_b : ClangBuiltin<"__builtin_ia32_pcmpgtb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_pcmpgt_w : GCCBuiltin<"__builtin_ia32_pcmpgtw">, + def int_x86_mmx_pcmpgt_w : ClangBuiltin<"__builtin_ia32_pcmpgtw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_pcmpgt_d : GCCBuiltin<"__builtin_ia32_pcmpgtd">, + def int_x86_mmx_pcmpgt_d : ClangBuiltin<"__builtin_ia32_pcmpgtd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; } // Misc. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_mmx_maskmovq : GCCBuiltin<"__builtin_ia32_maskmovq">, + def int_x86_mmx_maskmovq : ClangBuiltin<"__builtin_ia32_maskmovq">, Intrinsic<[], [llvm_x86mmx_ty, llvm_x86mmx_ty, llvm_ptr_ty], []>; - def int_x86_mmx_pmovmskb : GCCBuiltin<"__builtin_ia32_pmovmskb">, + def int_x86_mmx_pmovmskb : ClangBuiltin<"__builtin_ia32_pmovmskb">, Intrinsic<[llvm_i32_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_mmx_movnt_dq : GCCBuiltin<"__builtin_ia32_movntq">, + def int_x86_mmx_movnt_dq : ClangBuiltin<"__builtin_ia32_movntq">, Intrinsic<[], [llvm_ptrx86mmx_ty, llvm_x86mmx_ty], []>; - def int_x86_mmx_palignr_b : GCCBuiltin<"__builtin_ia32_palignr">, + def int_x86_mmx_palignr_b : ClangBuiltin<"__builtin_ia32_palignr">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_mmx_pextr_w : GCCBuiltin<"__builtin_ia32_vec_ext_v4hi">, + def int_x86_mmx_pextr_w : ClangBuiltin<"__builtin_ia32_vec_ext_v4hi">, Intrinsic<[llvm_i32_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_mmx_pinsr_w : GCCBuiltin<"__builtin_ia32_vec_set_v4hi">, + def int_x86_mmx_pinsr_w : ClangBuiltin<"__builtin_ia32_vec_set_v4hi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } @@ -2437,21 +2447,21 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // BMI let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_bmi_bextr_32 : GCCBuiltin<"__builtin_ia32_bextr_u32">, + def int_x86_bmi_bextr_32 : ClangBuiltin<"__builtin_ia32_bextr_u32">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_bmi_bextr_64 : GCCBuiltin<"__builtin_ia32_bextr_u64">, + def int_x86_bmi_bextr_64 : ClangBuiltin<"__builtin_ia32_bextr_u64">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_bmi_bzhi_32 : GCCBuiltin<"__builtin_ia32_bzhi_si">, + def int_x86_bmi_bzhi_32 : ClangBuiltin<"__builtin_ia32_bzhi_si">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_bmi_bzhi_64 : GCCBuiltin<"__builtin_ia32_bzhi_di">, + def int_x86_bmi_bzhi_64 : ClangBuiltin<"__builtin_ia32_bzhi_di">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_bmi_pdep_32 : GCCBuiltin<"__builtin_ia32_pdep_si">, + def int_x86_bmi_pdep_32 : ClangBuiltin<"__builtin_ia32_pdep_si">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_bmi_pdep_64 : GCCBuiltin<"__builtin_ia32_pdep_di">, + def int_x86_bmi_pdep_64 : ClangBuiltin<"__builtin_ia32_pdep_di">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_bmi_pext_32 : GCCBuiltin<"__builtin_ia32_pext_si">, + def int_x86_bmi_pext_32 : ClangBuiltin<"__builtin_ia32_pext_si">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_bmi_pext_64 : GCCBuiltin<"__builtin_ia32_pext_di">, + def int_x86_bmi_pext_64 : ClangBuiltin<"__builtin_ia32_pext_di">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; } @@ -2459,34 +2469,34 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FS/GS Base let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_rdfsbase_32 : GCCBuiltin<"__builtin_ia32_rdfsbase32">, + def int_x86_rdfsbase_32 : ClangBuiltin<"__builtin_ia32_rdfsbase32">, Intrinsic<[llvm_i32_ty], []>; - def int_x86_rdgsbase_32 : GCCBuiltin<"__builtin_ia32_rdgsbase32">, + def int_x86_rdgsbase_32 : ClangBuiltin<"__builtin_ia32_rdgsbase32">, Intrinsic<[llvm_i32_ty], []>; - def int_x86_rdfsbase_64 : GCCBuiltin<"__builtin_ia32_rdfsbase64">, + def int_x86_rdfsbase_64 : ClangBuiltin<"__builtin_ia32_rdfsbase64">, Intrinsic<[llvm_i64_ty], []>; - def int_x86_rdgsbase_64 : GCCBuiltin<"__builtin_ia32_rdgsbase64">, + def int_x86_rdgsbase_64 : ClangBuiltin<"__builtin_ia32_rdgsbase64">, Intrinsic<[llvm_i64_ty], []>; - def int_x86_wrfsbase_32 : GCCBuiltin<"__builtin_ia32_wrfsbase32">, + def int_x86_wrfsbase_32 : ClangBuiltin<"__builtin_ia32_wrfsbase32">, Intrinsic<[], [llvm_i32_ty]>; - def int_x86_wrgsbase_32 : GCCBuiltin<"__builtin_ia32_wrgsbase32">, + def int_x86_wrgsbase_32 : ClangBuiltin<"__builtin_ia32_wrgsbase32">, Intrinsic<[], [llvm_i32_ty]>; - def int_x86_wrfsbase_64 : GCCBuiltin<"__builtin_ia32_wrfsbase64">, + def int_x86_wrfsbase_64 : ClangBuiltin<"__builtin_ia32_wrfsbase64">, Intrinsic<[], [llvm_i64_ty]>; - def int_x86_wrgsbase_64 : GCCBuiltin<"__builtin_ia32_wrgsbase64">, + def int_x86_wrgsbase_64 : ClangBuiltin<"__builtin_ia32_wrgsbase64">, Intrinsic<[], [llvm_i64_ty]>; } //===----------------------------------------------------------------------===// // FXSR let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_fxrstor : GCCBuiltin<"__builtin_ia32_fxrstor">, + def int_x86_fxrstor : ClangBuiltin<"__builtin_ia32_fxrstor">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_fxrstor64 : GCCBuiltin<"__builtin_ia32_fxrstor64">, + def int_x86_fxrstor64 : ClangBuiltin<"__builtin_ia32_fxrstor64">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_fxsave : GCCBuiltin<"__builtin_ia32_fxsave">, + def int_x86_fxsave : ClangBuiltin<"__builtin_ia32_fxsave">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_fxsave64 : GCCBuiltin<"__builtin_ia32_fxsave64">, + def int_x86_fxsave64 : ClangBuiltin<"__builtin_ia32_fxsave64">, Intrinsic<[], [llvm_ptr_ty], []>; } @@ -2526,44 +2536,44 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". //===----------------------------------------------------------------------===// // CLFLUSHOPT and CLWB let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_clflushopt : GCCBuiltin<"__builtin_ia32_clflushopt">, + def int_x86_clflushopt : ClangBuiltin<"__builtin_ia32_clflushopt">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_clwb : GCCBuiltin<"__builtin_ia32_clwb">, + def int_x86_clwb : ClangBuiltin<"__builtin_ia32_clwb">, Intrinsic<[], [llvm_ptr_ty], []>; } //===----------------------------------------------------------------------===// // Support protection key let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_rdpkru : GCCBuiltin <"__builtin_ia32_rdpkru">, + def int_x86_rdpkru : ClangBuiltin <"__builtin_ia32_rdpkru">, Intrinsic<[llvm_i32_ty], [], []>; - def int_x86_wrpkru : GCCBuiltin<"__builtin_ia32_wrpkru">, + def int_x86_wrpkru : ClangBuiltin<"__builtin_ia32_wrpkru">, Intrinsic<[], [llvm_i32_ty], []>; } //===----------------------------------------------------------------------===// // Half float conversion let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph">, + def int_x86_vcvtps2ph_128 : ClangBuiltin<"__builtin_ia32_vcvtps2ph">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256">, + def int_x86_vcvtps2ph_256 : ClangBuiltin<"__builtin_ia32_vcvtps2ph256">, Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_vcvtph2ps_512 : Intrinsic<[llvm_v16f32_ty], [llvm_v16i16_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_mask_vcvtps2ph_512 : GCCBuiltin<"__builtin_ia32_vcvtps2ph512_mask">, + def int_x86_avx512_mask_vcvtps2ph_512 : ClangBuiltin<"__builtin_ia32_vcvtps2ph512_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256_mask">, + def int_x86_avx512_mask_vcvtps2ph_256 : ClangBuiltin<"__builtin_ia32_vcvtps2ph256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph_mask">, + def int_x86_avx512_mask_vcvtps2ph_128 : ClangBuiltin<"__builtin_ia32_vcvtps2ph_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; @@ -2573,10 +2583,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // TBM let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_tbm_bextri_u32 : GCCBuiltin<"__builtin_ia32_bextri_u32">, + def int_x86_tbm_bextri_u32 : ClangBuiltin<"__builtin_ia32_bextri_u32">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_tbm_bextri_u64 : GCCBuiltin<"__builtin_ia32_bextri_u64">, + def int_x86_tbm_bextri_u64 : ClangBuiltin<"__builtin_ia32_bextri_u64">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; } @@ -2619,13 +2629,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // RTM intrinsics. Transactional Memory support. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_xbegin : GCCBuiltin<"__builtin_ia32_xbegin">, + def int_x86_xbegin : ClangBuiltin<"__builtin_ia32_xbegin">, Intrinsic<[llvm_i32_ty], [], []>; - def int_x86_xend : GCCBuiltin<"__builtin_ia32_xend">, + def int_x86_xend : ClangBuiltin<"__builtin_ia32_xend">, Intrinsic<[], [], []>; - def int_x86_xabort : GCCBuiltin<"__builtin_ia32_xabort">, + def int_x86_xabort : ClangBuiltin<"__builtin_ia32_xabort">, Intrinsic<[], [llvm_i8_ty], [ImmArg<ArgIndex<0>>]>; - def int_x86_xtest : GCCBuiltin<"__builtin_ia32_xtest">, + def int_x86_xtest : ClangBuiltin<"__builtin_ia32_xtest">, Intrinsic<[llvm_i32_ty], [], []>; } @@ -2664,86 +2674,86 @@ let TargetPrefix = "x86" in { // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_cvttss2si : GCCBuiltin<"__builtin_ia32_vcvttss2si32">, + def int_x86_avx512_cvttss2si : ClangBuiltin<"__builtin_ia32_vcvttss2si32">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttss2si64 : GCCBuiltin<"__builtin_ia32_vcvttss2si64">, + def int_x86_avx512_cvttss2si64 : ClangBuiltin<"__builtin_ia32_vcvttss2si64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttss2usi : GCCBuiltin<"__builtin_ia32_vcvttss2usi32">, + def int_x86_avx512_cvttss2usi : ClangBuiltin<"__builtin_ia32_vcvttss2usi32">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_vcvttss2usi64">, + def int_x86_avx512_cvttss2usi64 : ClangBuiltin<"__builtin_ia32_vcvttss2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss32">, + def int_x86_avx512_cvtusi2ss : ClangBuiltin<"__builtin_ia32_cvtusi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss64">, + def int_x86_avx512_cvtusi642ss : ClangBuiltin<"__builtin_ia32_cvtusi2ss64">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_cvttsd2si : GCCBuiltin<"__builtin_ia32_vcvttsd2si32">, + def int_x86_avx512_cvttsd2si : ClangBuiltin<"__builtin_ia32_vcvttsd2si32">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_vcvttsd2si64">, + def int_x86_avx512_cvttsd2si64 : ClangBuiltin<"__builtin_ia32_vcvttsd2si64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttsd2usi : GCCBuiltin<"__builtin_ia32_vcvttsd2usi32">, + def int_x86_avx512_cvttsd2usi : ClangBuiltin<"__builtin_ia32_vcvttsd2usi32">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvttsd2usi64">, + def int_x86_avx512_cvttsd2usi64 : ClangBuiltin<"__builtin_ia32_vcvttsd2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd64">, + def int_x86_avx512_cvtusi642sd : ClangBuiltin<"__builtin_ia32_cvtusi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_vcvtss2usi32 : GCCBuiltin<"__builtin_ia32_vcvtss2usi32">, + def int_x86_avx512_vcvtss2usi32 : ClangBuiltin<"__builtin_ia32_vcvtss2usi32">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtss2usi64 : GCCBuiltin<"__builtin_ia32_vcvtss2usi64">, + def int_x86_avx512_vcvtss2usi64 : ClangBuiltin<"__builtin_ia32_vcvtss2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtss2si32 : GCCBuiltin<"__builtin_ia32_vcvtss2si32">, + def int_x86_avx512_vcvtss2si32 : ClangBuiltin<"__builtin_ia32_vcvtss2si32">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtss2si64 : GCCBuiltin<"__builtin_ia32_vcvtss2si64">, + def int_x86_avx512_vcvtss2si64 : ClangBuiltin<"__builtin_ia32_vcvtss2si64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtsd2usi32 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi32">, + def int_x86_avx512_vcvtsd2usi32 : ClangBuiltin<"__builtin_ia32_vcvtsd2usi32">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi64">, + def int_x86_avx512_vcvtsd2usi64 : ClangBuiltin<"__builtin_ia32_vcvtsd2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtsd2si32 : GCCBuiltin<"__builtin_ia32_vcvtsd2si32">, + def int_x86_avx512_vcvtsd2si32 : ClangBuiltin<"__builtin_ia32_vcvtsd2si32">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_vcvtsd2si64 : GCCBuiltin<"__builtin_ia32_vcvtsd2si64">, + def int_x86_avx512_vcvtsd2si64 : ClangBuiltin<"__builtin_ia32_vcvtsd2si64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">, + def int_x86_avx512_cvtsi2ss32 : ClangBuiltin<"__builtin_ia32_cvtsi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_cvtsi2ss64 : GCCBuiltin<"__builtin_ia32_cvtsi2ss64">, + def int_x86_avx512_cvtsi2ss64 : ClangBuiltin<"__builtin_ia32_cvtsi2ss64">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">, + def int_x86_avx512_cvtsi2sd64 : ClangBuiltin<"__builtin_ia32_cvtsi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; } // Pack ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_packsswb_512 : GCCBuiltin<"__builtin_ia32_packsswb512">, + def int_x86_avx512_packsswb_512 : ClangBuiltin<"__builtin_ia32_packsswb512">, Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_packssdw_512 : GCCBuiltin<"__builtin_ia32_packssdw512">, + def int_x86_avx512_packssdw_512 : ClangBuiltin<"__builtin_ia32_packssdw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; - def int_x86_avx512_packuswb_512 : GCCBuiltin<"__builtin_ia32_packuswb512">, + def int_x86_avx512_packuswb_512 : ClangBuiltin<"__builtin_ia32_packuswb512">, Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_packusdw_512 : GCCBuiltin<"__builtin_ia32_packusdw512">, + def int_x86_avx512_packusdw_512 : ClangBuiltin<"__builtin_ia32_packusdw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; } @@ -2759,380 +2769,380 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_cvtpd2dq_128 : - GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2dq_512 : - GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtpd2ps_512 : - GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtsd2ss_round : - GCCBuiltin<"__builtin_ia32_cvtsd2ss_round_mask">, + ClangBuiltin<"__builtin_ia32_cvtsd2ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_cvtss2sd_round : - GCCBuiltin<"__builtin_ia32_cvtss2sd_round_mask">, + ClangBuiltin<"__builtin_ia32_cvtss2sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_cvtpd2ps : - GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2ps_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2qq_128 : - GCCBuiltin<"__builtin_ia32_cvtpd2qq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2qq_256 : - GCCBuiltin<"__builtin_ia32_cvtpd2qq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2qq_512 : - GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtpd2udq_128 : - GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2udq_256 : - GCCBuiltin<"__builtin_ia32_cvtpd2udq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2udq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2udq_512 : - GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtpd2uqq_128 : - GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2uqq_256 : - GCCBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtpd2uqq_512 : - GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtps2dq_128 : - GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2dq_256 : - GCCBuiltin<"__builtin_ia32_cvtps2dq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2dq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2dq_512 : - GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtps2pd_512 : - GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtps2qq_128 : - GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2qq_256 : - GCCBuiltin<"__builtin_ia32_cvtps2qq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2qq_512 : - GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtps2udq_128 : - GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2udq_256 : - GCCBuiltin<"__builtin_ia32_cvtps2udq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2udq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2udq_512 : - GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtps2uqq_128 : - GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2uqq_256 : - GCCBuiltin<"__builtin_ia32_cvtps2uqq256_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvtps2uqq_512 : - GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">, + ClangBuiltin<"__builtin_ia32_cvtps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtqq2ps_128 : - GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">, + ClangBuiltin<"__builtin_ia32_cvtqq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2dq_128 : - GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2dq_512 : - GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttpd2qq_128 : - GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2qq_256 : - GCCBuiltin<"__builtin_ia32_cvttpd2qq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2qq_512 : - GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttpd2udq_128 : - GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2udq_256 : - GCCBuiltin<"__builtin_ia32_cvttpd2udq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2udq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2udq_512 : - GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttpd2uqq_128 : - GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2uqq_256 : - GCCBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttpd2uqq_512 : - GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttps2dq_512 : - GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttps2qq_128 : - GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2qq_256 : - GCCBuiltin<"__builtin_ia32_cvttps2qq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2qq_512 : - GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttps2udq_128 : - GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2udq_256 : - GCCBuiltin<"__builtin_ia32_cvttps2udq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2udq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2udq_512 : - GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvttps2uqq_128 : - GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2uqq_256 : - GCCBuiltin<"__builtin_ia32_cvttps2uqq256_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_cvttps2uqq_512 : - GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">, + ClangBuiltin<"__builtin_ia32_cvttps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_cvtuqq2ps_128 : - GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">, + ClangBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">, + def int_x86_avx512_mask_rndscale_pd_128 : ClangBuiltin<"__builtin_ia32_rndscalepd_128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">, + def int_x86_avx512_mask_rndscale_pd_256 : ClangBuiltin<"__builtin_ia32_rndscalepd_256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">, + def int_x86_avx512_mask_rndscale_pd_512 : ClangBuiltin<"__builtin_ia32_rndscalepd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">, + def int_x86_avx512_mask_rndscale_ps_128 : ClangBuiltin<"__builtin_ia32_rndscaleps_128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">, + def int_x86_avx512_mask_rndscale_ps_256 : ClangBuiltin<"__builtin_ia32_rndscaleps_256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">, + def int_x86_avx512_mask_rndscale_ps_512 : ClangBuiltin<"__builtin_ia32_rndscaleps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">, + def int_x86_avx512_mask_reduce_pd_128 : ClangBuiltin<"__builtin_ia32_reducepd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">, + def int_x86_avx512_mask_reduce_pd_256 : ClangBuiltin<"__builtin_ia32_reducepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_reduce_pd_512 : GCCBuiltin<"__builtin_ia32_reducepd512_mask">, + def int_x86_avx512_mask_reduce_pd_512 : ClangBuiltin<"__builtin_ia32_reducepd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">, + def int_x86_avx512_mask_reduce_ps_128 : ClangBuiltin<"__builtin_ia32_reduceps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">, + def int_x86_avx512_mask_reduce_ps_256 : ClangBuiltin<"__builtin_ia32_reduceps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; - def int_x86_avx512_mask_reduce_ps_512 : GCCBuiltin<"__builtin_ia32_reduceps512_mask">, + def int_x86_avx512_mask_reduce_ps_512 : ClangBuiltin<"__builtin_ia32_reduceps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; -def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">, +def int_x86_avx512_mask_range_pd_128 : ClangBuiltin<"__builtin_ia32_rangepd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">, +def int_x86_avx512_mask_range_pd_256 : ClangBuiltin<"__builtin_ia32_rangepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_x86_avx512_mask_range_pd_512 : GCCBuiltin<"__builtin_ia32_rangepd512_mask">, +def int_x86_avx512_mask_range_pd_512 : ClangBuiltin<"__builtin_ia32_rangepd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>]>; -def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mask">, +def int_x86_avx512_mask_range_ps_128 : ClangBuiltin<"__builtin_ia32_rangeps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">, +def int_x86_avx512_mask_range_ps_256 : ClangBuiltin<"__builtin_ia32_rangeps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; -def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">, +def int_x86_avx512_mask_range_ps_512 : ClangBuiltin<"__builtin_ia32_rangeps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>]>; @@ -3141,152 +3151,152 @@ def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mas // Vector load with broadcast let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_broadcastmw_512 : - GCCBuiltin<"__builtin_ia32_broadcastmw512">, + ClangBuiltin<"__builtin_ia32_broadcastmw512">, Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmw_256 : - GCCBuiltin<"__builtin_ia32_broadcastmw256">, + ClangBuiltin<"__builtin_ia32_broadcastmw256">, Intrinsic<[llvm_v8i32_ty], [llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmw_128 : - GCCBuiltin<"__builtin_ia32_broadcastmw128">, + ClangBuiltin<"__builtin_ia32_broadcastmw128">, Intrinsic<[llvm_v4i32_ty], [llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmb_512 : - GCCBuiltin<"__builtin_ia32_broadcastmb512">, + ClangBuiltin<"__builtin_ia32_broadcastmb512">, Intrinsic<[llvm_v8i64_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmb_256 : - GCCBuiltin<"__builtin_ia32_broadcastmb256">, + ClangBuiltin<"__builtin_ia32_broadcastmb256">, Intrinsic<[llvm_v4i64_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmb_128 : - GCCBuiltin<"__builtin_ia32_broadcastmb128">, + ClangBuiltin<"__builtin_ia32_broadcastmb128">, Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>; } // Arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_add_ps_512 : GCCBuiltin<"__builtin_ia32_addps512">, + def int_x86_avx512_add_ps_512 : ClangBuiltin<"__builtin_ia32_addps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_add_pd_512 : GCCBuiltin<"__builtin_ia32_addpd512">, + def int_x86_avx512_add_pd_512 : ClangBuiltin<"__builtin_ia32_addpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_sub_ps_512 : GCCBuiltin<"__builtin_ia32_subps512">, + def int_x86_avx512_sub_ps_512 : ClangBuiltin<"__builtin_ia32_subps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_sub_pd_512 : GCCBuiltin<"__builtin_ia32_subpd512">, + def int_x86_avx512_sub_pd_512 : ClangBuiltin<"__builtin_ia32_subpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_mul_ps_512 : GCCBuiltin<"__builtin_ia32_mulps512">, + def int_x86_avx512_mul_ps_512 : ClangBuiltin<"__builtin_ia32_mulps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_mul_pd_512 : GCCBuiltin<"__builtin_ia32_mulpd512">, + def int_x86_avx512_mul_pd_512 : ClangBuiltin<"__builtin_ia32_mulpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_div_ps_512 : GCCBuiltin<"__builtin_ia32_divps512">, + def int_x86_avx512_div_ps_512 : ClangBuiltin<"__builtin_ia32_divps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_div_pd_512 : GCCBuiltin<"__builtin_ia32_divpd512">, + def int_x86_avx512_div_pd_512 : ClangBuiltin<"__builtin_ia32_divpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_max_ps_512 : GCCBuiltin<"__builtin_ia32_maxps512">, + def int_x86_avx512_max_ps_512 : ClangBuiltin<"__builtin_ia32_maxps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_max_pd_512 : GCCBuiltin<"__builtin_ia32_maxpd512">, + def int_x86_avx512_max_pd_512 : ClangBuiltin<"__builtin_ia32_maxpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_min_ps_512 : GCCBuiltin<"__builtin_ia32_minps512">, + def int_x86_avx512_min_ps_512 : ClangBuiltin<"__builtin_ia32_minps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_min_pd_512 : GCCBuiltin<"__builtin_ia32_minpd512">, + def int_x86_avx512_min_pd_512 : ClangBuiltin<"__builtin_ia32_minpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_round_mask">, + def int_x86_avx512_mask_add_ss_round : ClangBuiltin<"__builtin_ia32_addss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_round_mask">, + def int_x86_avx512_mask_div_ss_round : ClangBuiltin<"__builtin_ia32_divss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_round_mask">, + def int_x86_avx512_mask_mul_ss_round : ClangBuiltin<"__builtin_ia32_mulss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_round_mask">, + def int_x86_avx512_mask_sub_ss_round : ClangBuiltin<"__builtin_ia32_subss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_round_mask">, + def int_x86_avx512_mask_max_ss_round : ClangBuiltin<"__builtin_ia32_maxss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_round_mask">, + def int_x86_avx512_mask_min_ss_round : ClangBuiltin<"__builtin_ia32_minss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_round_mask">, + def int_x86_avx512_mask_add_sd_round : ClangBuiltin<"__builtin_ia32_addsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_round_mask">, + def int_x86_avx512_mask_div_sd_round : ClangBuiltin<"__builtin_ia32_divsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_round_mask">, + def int_x86_avx512_mask_mul_sd_round : ClangBuiltin<"__builtin_ia32_mulsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round_mask">, + def int_x86_avx512_mask_sub_sd_round : ClangBuiltin<"__builtin_ia32_subsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round_mask">, + def int_x86_avx512_mask_max_sd_round : ClangBuiltin<"__builtin_ia32_maxsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round_mask">, + def int_x86_avx512_mask_min_sd_round : ClangBuiltin<"__builtin_ia32_minsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round_mask">, + def int_x86_avx512_mask_rndscale_ss : ClangBuiltin<"__builtin_ia32_rndscaless_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round_mask">, + def int_x86_avx512_mask_rndscale_sd : ClangBuiltin<"__builtin_ia32_rndscalesd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round_mask">, + def int_x86_avx512_mask_range_ss : ClangBuiltin<"__builtin_ia32_rangess128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round_mask">, + def int_x86_avx512_mask_range_sd : ClangBuiltin<"__builtin_ia32_rangesd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess_mask">, + def int_x86_avx512_mask_reduce_ss : ClangBuiltin<"__builtin_ia32_reducess_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd_mask">, + def int_x86_avx512_mask_reduce_sd : ClangBuiltin<"__builtin_ia32_reducesd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round_mask">, + def int_x86_avx512_mask_scalef_sd : ClangBuiltin<"__builtin_ia32_scalefsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round_mask">, + def int_x86_avx512_mask_scalef_ss : ClangBuiltin<"__builtin_ia32_scalefss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">, + def int_x86_avx512_mask_scalef_pd_128 : ClangBuiltin<"__builtin_ia32_scalefpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_scalef_pd_256 : GCCBuiltin<"__builtin_ia32_scalefpd256_mask">, + def int_x86_avx512_mask_scalef_pd_256 : ClangBuiltin<"__builtin_ia32_scalefpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty],[IntrNoMem]>; - def int_x86_avx512_mask_scalef_pd_512 : GCCBuiltin<"__builtin_ia32_scalefpd512_mask">, + def int_x86_avx512_mask_scalef_pd_512 : ClangBuiltin<"__builtin_ia32_scalefpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_scalef_ps_128 : GCCBuiltin<"__builtin_ia32_scalefps128_mask">, + def int_x86_avx512_mask_scalef_ps_128 : ClangBuiltin<"__builtin_ia32_scalefps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_scalef_ps_256 : GCCBuiltin<"__builtin_ia32_scalefps256_mask">, + def int_x86_avx512_mask_scalef_ps_256 : ClangBuiltin<"__builtin_ia32_scalefps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_scalef_ps_512 : GCCBuiltin<"__builtin_ia32_scalefps512_mask">, + def int_x86_avx512_mask_scalef_ps_512 : ClangBuiltin<"__builtin_ia32_scalefps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; @@ -3307,290 +3317,290 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_fixupimm_pd_128 : - GCCBuiltin<"__builtin_ia32_fixupimmpd128_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_maskz_fixupimm_pd_128 : - GCCBuiltin<"__builtin_ia32_fixupimmpd128_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmpd128_maskz">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_fixupimm_pd_256 : - GCCBuiltin<"__builtin_ia32_fixupimmpd256_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_maskz_fixupimm_pd_256 : - GCCBuiltin<"__builtin_ia32_fixupimmpd256_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmpd256_maskz">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_fixupimm_pd_512 : - GCCBuiltin<"__builtin_ia32_fixupimmpd512_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_maskz_fixupimm_pd_512 : - GCCBuiltin<"__builtin_ia32_fixupimmpd512_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmpd512_maskz">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_mask_fixupimm_ps_128 : - GCCBuiltin<"__builtin_ia32_fixupimmps128_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_maskz_fixupimm_ps_128 : - GCCBuiltin<"__builtin_ia32_fixupimmps128_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmps128_maskz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_fixupimm_ps_256 : - GCCBuiltin<"__builtin_ia32_fixupimmps256_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_maskz_fixupimm_ps_256 : - GCCBuiltin<"__builtin_ia32_fixupimmps256_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmps256_maskz">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_mask_fixupimm_ps_512 : - GCCBuiltin<"__builtin_ia32_fixupimmps512_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_maskz_fixupimm_ps_512 : - GCCBuiltin<"__builtin_ia32_fixupimmps512_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmps512_maskz">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_mask_fixupimm_sd : - GCCBuiltin<"__builtin_ia32_fixupimmsd_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmsd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_maskz_fixupimm_sd : - GCCBuiltin<"__builtin_ia32_fixupimmsd_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmsd_maskz">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_mask_fixupimm_ss : - GCCBuiltin<"__builtin_ia32_fixupimmss_mask">, + ClangBuiltin<"__builtin_ia32_fixupimmss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_maskz_fixupimm_ss : - GCCBuiltin<"__builtin_ia32_fixupimmss_maskz">, + ClangBuiltin<"__builtin_ia32_fixupimmss_maskz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_mask_getexp_pd_128 : GCCBuiltin<"__builtin_ia32_getexppd128_mask">, + def int_x86_avx512_mask_getexp_pd_128 : ClangBuiltin<"__builtin_ia32_getexppd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_pd_256 : GCCBuiltin<"__builtin_ia32_getexppd256_mask">, + def int_x86_avx512_mask_getexp_pd_256 : ClangBuiltin<"__builtin_ia32_getexppd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_pd_512 : GCCBuiltin<"__builtin_ia32_getexppd512_mask">, + def int_x86_avx512_mask_getexp_pd_512 : ClangBuiltin<"__builtin_ia32_getexppd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_mask_getexp_ps_128 : GCCBuiltin<"__builtin_ia32_getexpps128_mask">, + def int_x86_avx512_mask_getexp_ps_128 : ClangBuiltin<"__builtin_ia32_getexpps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_ps_256 : GCCBuiltin<"__builtin_ia32_getexpps256_mask">, + def int_x86_avx512_mask_getexp_ps_256 : ClangBuiltin<"__builtin_ia32_getexpps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_ps_512 : GCCBuiltin<"__builtin_ia32_getexpps512_mask">, + def int_x86_avx512_mask_getexp_ps_512 : ClangBuiltin<"__builtin_ia32_getexpps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss128_round_mask">, + def int_x86_avx512_mask_getexp_ss : ClangBuiltin<"__builtin_ia32_getexpss128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd128_round_mask">, + def int_x86_avx512_mask_getexp_sd : ClangBuiltin<"__builtin_ia32_getexpsd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_getmant_pd_128 : - GCCBuiltin<"__builtin_ia32_getmantpd128_mask">, + ClangBuiltin<"__builtin_ia32_getmantpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_getmant_pd_256 : - GCCBuiltin<"__builtin_ia32_getmantpd256_mask">, + ClangBuiltin<"__builtin_ia32_getmantpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_getmant_pd_512 : - GCCBuiltin<"__builtin_ia32_getmantpd512_mask">, + ClangBuiltin<"__builtin_ia32_getmantpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty,llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty,llvm_i32_ty ], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_getmant_ps_128 : - GCCBuiltin<"__builtin_ia32_getmantps128_mask">, + ClangBuiltin<"__builtin_ia32_getmantps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_getmant_ps_256 : - GCCBuiltin<"__builtin_ia32_getmantps256_mask">, + ClangBuiltin<"__builtin_ia32_getmantps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>; def int_x86_avx512_mask_getmant_ps_512 : - GCCBuiltin<"__builtin_ia32_getmantps512_mask">, + ClangBuiltin<"__builtin_ia32_getmantps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty,llvm_i32_ty, llvm_v16f32_ty,llvm_i16_ty,llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_getmant_ss : - GCCBuiltin<"__builtin_ia32_getmantss_round_mask">, + ClangBuiltin<"__builtin_ia32_getmantss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>]>; def int_x86_avx512_mask_getmant_sd : - GCCBuiltin<"__builtin_ia32_getmantsd_round_mask">, + ClangBuiltin<"__builtin_ia32_getmantsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>>]>; - def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">, + def int_x86_avx512_rsqrt14_ss : ClangBuiltin<"__builtin_ia32_rsqrt14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd_mask">, + def int_x86_avx512_rsqrt14_sd : ClangBuiltin<"__builtin_ia32_rsqrt14sd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_pd_128 : GCCBuiltin<"__builtin_ia32_rsqrt14pd128_mask">, + def int_x86_avx512_rsqrt14_pd_128 : ClangBuiltin<"__builtin_ia32_rsqrt14pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_pd_256 : GCCBuiltin<"__builtin_ia32_rsqrt14pd256_mask">, + def int_x86_avx512_rsqrt14_pd_256 : ClangBuiltin<"__builtin_ia32_rsqrt14pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_pd_512 : GCCBuiltin<"__builtin_ia32_rsqrt14pd512_mask">, + def int_x86_avx512_rsqrt14_pd_512 : ClangBuiltin<"__builtin_ia32_rsqrt14pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_ps_128 : GCCBuiltin<"__builtin_ia32_rsqrt14ps128_mask">, + def int_x86_avx512_rsqrt14_ps_128 : ClangBuiltin<"__builtin_ia32_rsqrt14ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrt14ps256_mask">, + def int_x86_avx512_rsqrt14_ps_256 : ClangBuiltin<"__builtin_ia32_rsqrt14ps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_ps_512 : GCCBuiltin<"__builtin_ia32_rsqrt14ps512_mask">, + def int_x86_avx512_rsqrt14_ps_512 : ClangBuiltin<"__builtin_ia32_rsqrt14ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss_mask">, + def int_x86_avx512_rcp14_ss : ClangBuiltin<"__builtin_ia32_rcp14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd_mask">, + def int_x86_avx512_rcp14_sd : ClangBuiltin<"__builtin_ia32_rcp14sd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_pd_128 : GCCBuiltin<"__builtin_ia32_rcp14pd128_mask">, + def int_x86_avx512_rcp14_pd_128 : ClangBuiltin<"__builtin_ia32_rcp14pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_pd_256 : GCCBuiltin<"__builtin_ia32_rcp14pd256_mask">, + def int_x86_avx512_rcp14_pd_256 : ClangBuiltin<"__builtin_ia32_rcp14pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_pd_512 : GCCBuiltin<"__builtin_ia32_rcp14pd512_mask">, + def int_x86_avx512_rcp14_pd_512 : ClangBuiltin<"__builtin_ia32_rcp14pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ps_128 : GCCBuiltin<"__builtin_ia32_rcp14ps128_mask">, + def int_x86_avx512_rcp14_ps_128 : ClangBuiltin<"__builtin_ia32_rcp14ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ps_256 : GCCBuiltin<"__builtin_ia32_rcp14ps256_mask">, + def int_x86_avx512_rcp14_ps_256 : ClangBuiltin<"__builtin_ia32_rcp14ps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ps_512 : GCCBuiltin<"__builtin_ia32_rcp14ps512_mask">, + def int_x86_avx512_rcp14_ps_512 : ClangBuiltin<"__builtin_ia32_rcp14ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_rcp28_ps : GCCBuiltin<"__builtin_ia32_rcp28ps_mask">, + def int_x86_avx512_rcp28_ps : ClangBuiltin<"__builtin_ia32_rcp28ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_rcp28_pd : GCCBuiltin<"__builtin_ia32_rcp28pd_mask">, + def int_x86_avx512_rcp28_pd : ClangBuiltin<"__builtin_ia32_rcp28pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_exp2_ps : GCCBuiltin<"__builtin_ia32_exp2ps_mask">, + def int_x86_avx512_exp2_ps : ClangBuiltin<"__builtin_ia32_exp2ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_exp2_pd : GCCBuiltin<"__builtin_ia32_exp2pd_mask">, + def int_x86_avx512_exp2_pd : ClangBuiltin<"__builtin_ia32_exp2pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round_mask">, + def int_x86_avx512_rcp28_ss : ClangBuiltin<"__builtin_ia32_rcp28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round_mask">, + def int_x86_avx512_rcp28_sd : ClangBuiltin<"__builtin_ia32_rcp28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_rsqrt28_ps : GCCBuiltin<"__builtin_ia32_rsqrt28ps_mask">, + def int_x86_avx512_rsqrt28_ps : ClangBuiltin<"__builtin_ia32_rsqrt28ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_rsqrt28_pd : GCCBuiltin<"__builtin_ia32_rsqrt28pd_mask">, + def int_x86_avx512_rsqrt28_pd : ClangBuiltin<"__builtin_ia32_rsqrt28pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round_mask">, + def int_x86_avx512_rsqrt28_ss : ClangBuiltin<"__builtin_ia32_rsqrt28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round_mask">, + def int_x86_avx512_rsqrt28_sd : ClangBuiltin<"__builtin_ia32_rsqrt28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_psad_bw_512 : GCCBuiltin<"__builtin_ia32_psadbw512">, + def int_x86_avx512_psad_bw_512 : ClangBuiltin<"__builtin_ia32_psadbw512">, Intrinsic<[llvm_v8i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem, Commutative]>; } // Integer arithmetic ops let TargetPrefix = "x86" in { - def int_x86_avx512_pmulhu_w_512 : GCCBuiltin<"__builtin_ia32_pmulhuw512">, + def int_x86_avx512_pmulhu_w_512 : ClangBuiltin<"__builtin_ia32_pmulhuw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx512_pmulh_w_512 : GCCBuiltin<"__builtin_ia32_pmulhw512">, + def int_x86_avx512_pmulh_w_512 : ClangBuiltin<"__builtin_ia32_pmulhw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx512_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512">, + def int_x86_avx512_pavg_b_512 : ClangBuiltin<"__builtin_ia32_pavgb512">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; - def int_x86_avx512_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512">, + def int_x86_avx512_pavg_w_512 : ClangBuiltin<"__builtin_ia32_pavgw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem]>; - def int_x86_avx512_pmaddw_d_512 : GCCBuiltin<"__builtin_ia32_pmaddwd512">, + def int_x86_avx512_pmaddw_d_512 : ClangBuiltin<"__builtin_ia32_pmaddwd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx512_pmaddubs_w_512 : GCCBuiltin<"__builtin_ia32_pmaddubsw512">, + def int_x86_avx512_pmaddubs_w_512 : ClangBuiltin<"__builtin_ia32_pmaddubsw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx512_dbpsadbw_128 : - GCCBuiltin<"__builtin_ia32_dbpsadbw128">, + ClangBuiltin<"__builtin_ia32_dbpsadbw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_avx512_dbpsadbw_256 : - GCCBuiltin<"__builtin_ia32_dbpsadbw256">, + ClangBuiltin<"__builtin_ia32_dbpsadbw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_avx512_dbpsadbw_512 : - GCCBuiltin<"__builtin_ia32_dbpsadbw512">, + ClangBuiltin<"__builtin_ia32_dbpsadbw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; @@ -3838,32 +3848,32 @@ let TargetPrefix = "x86" in { // gather prefetch // NOTE: These can't be ArgMemOnly because you can put the address completely // in the index register. - def int_x86_avx512_gatherpf_dpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfdpd">, + def int_x86_avx512_gatherpf_dpd_512 : ClangBuiltin<"__builtin_ia32_gatherpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_gatherpf_dps_512 : GCCBuiltin<"__builtin_ia32_gatherpfdps">, + def int_x86_avx512_gatherpf_dps_512 : ClangBuiltin<"__builtin_ia32_gatherpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_gatherpf_qpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfqpd">, + def int_x86_avx512_gatherpf_qpd_512 : ClangBuiltin<"__builtin_ia32_gatherpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_gatherpf_qps_512 : GCCBuiltin<"__builtin_ia32_gatherpfqps">, + def int_x86_avx512_gatherpf_qps_512 : ClangBuiltin<"__builtin_ia32_gatherpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; // scatter prefetch // NOTE: These can't be ArgMemOnly because you can put the address completely // in the index register. - def int_x86_avx512_scatterpf_dpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfdpd">, + def int_x86_avx512_scatterpf_dpd_512 : ClangBuiltin<"__builtin_ia32_scatterpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_scatterpf_dps_512 : GCCBuiltin<"__builtin_ia32_scatterpfdps">, + def int_x86_avx512_scatterpf_dps_512 : ClangBuiltin<"__builtin_ia32_scatterpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_scatterpf_qpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfqpd">, + def int_x86_avx512_scatterpf_qpd_512 : ClangBuiltin<"__builtin_ia32_scatterpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; - def int_x86_avx512_scatterpf_qps_512 : GCCBuiltin<"__builtin_ia32_scatterpfqps">, + def int_x86_avx512_scatterpf_qps_512 : ClangBuiltin<"__builtin_ia32_scatterpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; } @@ -4109,34 +4119,34 @@ let TargetPrefix = "x86" in { // Instructions that count the number of leading zero bits let TargetPrefix = "x86" in { def int_x86_avx512_conflict_d_128 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_128">, + ClangBuiltin<"__builtin_ia32_vpconflictsi_128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_conflict_d_256 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_256">, + ClangBuiltin<"__builtin_ia32_vpconflictsi_256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_conflict_d_512 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_512">, + ClangBuiltin<"__builtin_ia32_vpconflictsi_512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_conflict_q_128 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_128">, + ClangBuiltin<"__builtin_ia32_vpconflictdi_128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_conflict_q_256 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_256">, + ClangBuiltin<"__builtin_ia32_vpconflictdi_256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx512_conflict_q_512 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_512">, + ClangBuiltin<"__builtin_ia32_vpconflictdi_512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty], [IntrNoMem]>; } // Compares let TargetPrefix = "x86" in { // 512-bit - def int_x86_avx512_vcomi_sd : GCCBuiltin<"__builtin_ia32_vcomisd">, + def int_x86_avx512_vcomi_sd : ClangBuiltin<"__builtin_ia32_vcomisd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>; - def int_x86_avx512_vcomi_ss : GCCBuiltin<"__builtin_ia32_vcomiss">, + def int_x86_avx512_vcomi_ss : ClangBuiltin<"__builtin_ia32_vcomiss">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>; @@ -4159,152 +4169,152 @@ let TargetPrefix = "x86" in { // truncate let TargetPrefix = "x86" in { def int_x86_avx512_mask_pmov_qb_128 : - GCCBuiltin<"__builtin_ia32_pmovqb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovqb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_128 : - GCCBuiltin<"__builtin_ia32_pmovsqb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovsqb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_128 : - GCCBuiltin<"__builtin_ia32_pmovusqb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovusqb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qb_256 : - GCCBuiltin<"__builtin_ia32_pmovqb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovqb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_256 : - GCCBuiltin<"__builtin_ia32_pmovsqb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovsqb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_256 : - GCCBuiltin<"__builtin_ia32_pmovusqb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovusqb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qb_512 : - GCCBuiltin<"__builtin_ia32_pmovqb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb512_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovqb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_512 : - GCCBuiltin<"__builtin_ia32_pmovsqb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb512_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovsqb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_512 : - GCCBuiltin<"__builtin_ia32_pmovusqb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb512_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovusqb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qw_128 : - GCCBuiltin<"__builtin_ia32_pmovqw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovqw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovqw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_128 : - GCCBuiltin<"__builtin_ia32_pmovsqw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovsqw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_128 : - GCCBuiltin<"__builtin_ia32_pmovusqw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovusqw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qw_256 : - GCCBuiltin<"__builtin_ia32_pmovqw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovqw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovqw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_256 : - GCCBuiltin<"__builtin_ia32_pmovsqw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovsqw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_256 : - GCCBuiltin<"__builtin_ia32_pmovusqw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovusqw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; @@ -4313,167 +4323,167 @@ let TargetPrefix = "x86" in { [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovqw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_512 : - GCCBuiltin<"__builtin_ia32_pmovsqw512_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw512_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovsqw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_512 : - GCCBuiltin<"__builtin_ia32_pmovusqw512_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw512_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovusqw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_128 : - GCCBuiltin<"__builtin_ia32_pmovqd128_mask">, + ClangBuiltin<"__builtin_ia32_pmovqd128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_qd_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovqd128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_128 : - GCCBuiltin<"__builtin_ia32_pmovsqd128_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qd_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovsqd128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_128 : - GCCBuiltin<"__builtin_ia32_pmovusqd128_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qd_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovusqd128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_256 : - GCCBuiltin<"__builtin_ia32_pmovsqd256_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qd_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovsqd256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_256 : - GCCBuiltin<"__builtin_ia32_pmovusqd256_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qd_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovusqd256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_512 : - GCCBuiltin<"__builtin_ia32_pmovsqd512_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_qd_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovsqd512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_512 : - GCCBuiltin<"__builtin_ia32_pmovusqd512_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_qd_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovusqd512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_db_128 : - GCCBuiltin<"__builtin_ia32_pmovdb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovdb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_db_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovdb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_128 : - GCCBuiltin<"__builtin_ia32_pmovsdb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_db_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovsdb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_128 : - GCCBuiltin<"__builtin_ia32_pmovusdb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_db_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovusdb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_db_256 : - GCCBuiltin<"__builtin_ia32_pmovdb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovdb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_db_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovdb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_256 : - GCCBuiltin<"__builtin_ia32_pmovsdb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_db_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovsdb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_256 : - GCCBuiltin<"__builtin_ia32_pmovusdb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_db_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovusdb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; @@ -4482,87 +4492,87 @@ let TargetPrefix = "x86" in { [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_db_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovdb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_512 : - GCCBuiltin<"__builtin_ia32_pmovsdb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb512_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_db_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovsdb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_512 : - GCCBuiltin<"__builtin_ia32_pmovusdb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb512_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_db_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovusdb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_dw_128 : - GCCBuiltin<"__builtin_ia32_pmovdw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovdw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_dw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovdw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_128 : - GCCBuiltin<"__builtin_ia32_pmovsdw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_dw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovsdw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_128 : - GCCBuiltin<"__builtin_ia32_pmovusdw128_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_dw_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovusdw128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_dw_256 : - GCCBuiltin<"__builtin_ia32_pmovdw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovdw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_dw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovdw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_256 : - GCCBuiltin<"__builtin_ia32_pmovsdw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_dw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovsdw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_256 : - GCCBuiltin<"__builtin_ia32_pmovusdw256_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_dw_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovusdw256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrArgMemOnly]>; @@ -4571,107 +4581,107 @@ let TargetPrefix = "x86" in { [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_dw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovdw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_512 : - GCCBuiltin<"__builtin_ia32_pmovsdw512_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw512_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_dw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovsdw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovsdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_512 : - GCCBuiltin<"__builtin_ia32_pmovusdw512_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw512_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_dw_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovusdw512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovusdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_128 : - GCCBuiltin<"__builtin_ia32_pmovwb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovwb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmov_wb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovwb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovwb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_128 : - GCCBuiltin<"__builtin_ia32_pmovswb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_wb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovswb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_128 : - GCCBuiltin<"__builtin_ia32_pmovuswb128_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_wb_mem_128 : - GCCBuiltin<"__builtin_ia32_pmovuswb128mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovwb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_256 : - GCCBuiltin<"__builtin_ia32_pmovswb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_wb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovswb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_256 : - GCCBuiltin<"__builtin_ia32_pmovuswb256_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb256_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_wb_mem_256 : - GCCBuiltin<"__builtin_ia32_pmovuswb256mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovwb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_512 : - GCCBuiltin<"__builtin_ia32_pmovswb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb512_mask">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovs_wb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovswb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovswb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_512 : - GCCBuiltin<"__builtin_ia32_pmovuswb512_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb512_mask">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmovus_wb_mem_512 : - GCCBuiltin<"__builtin_ia32_pmovuswb512mem_mask">, + ClangBuiltin<"__builtin_ia32_pmovuswb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrArgMemOnly]>; @@ -4680,37 +4690,37 @@ let TargetPrefix = "x86" in { // Bitwise ternary logic let TargetPrefix = "x86" in { def int_x86_avx512_pternlog_d_128 : - GCCBuiltin<"__builtin_ia32_pternlogd128">, + ClangBuiltin<"__builtin_ia32_pternlogd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_pternlog_d_256 : - GCCBuiltin<"__builtin_ia32_pternlogd256">, + ClangBuiltin<"__builtin_ia32_pternlogd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_pternlog_d_512 : - GCCBuiltin<"__builtin_ia32_pternlogd512">, + ClangBuiltin<"__builtin_ia32_pternlogd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_pternlog_q_128 : - GCCBuiltin<"__builtin_ia32_pternlogq128">, + ClangBuiltin<"__builtin_ia32_pternlogq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_pternlog_q_256 : - GCCBuiltin<"__builtin_ia32_pternlogq256">, + ClangBuiltin<"__builtin_ia32_pternlogq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; def int_x86_avx512_pternlog_q_512 : - GCCBuiltin<"__builtin_ia32_pternlogq512">, + ClangBuiltin<"__builtin_ia32_pternlogq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>]>; @@ -4770,12 +4780,12 @@ let TargetPrefix = "x86" in { llvm_i32_ty, llvm_v2i1_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; def int_x86_avx512_mask_cmp_ss : - GCCBuiltin<"__builtin_ia32_cmpss_mask">, + ClangBuiltin<"__builtin_ia32_cmpss_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<4>>]>; def int_x86_avx512_mask_cmp_sd : - GCCBuiltin<"__builtin_ia32_cmpsd_mask">, + ClangBuiltin<"__builtin_ia32_cmpsd_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<4>>]>; @@ -4784,21 +4794,21 @@ let TargetPrefix = "x86" in { //===----------------------------------------------------------------------===// // SHA intrinsics let TargetPrefix = "x86" in { - def int_x86_sha1rnds4 : GCCBuiltin<"__builtin_ia32_sha1rnds4">, + def int_x86_sha1rnds4 : ClangBuiltin<"__builtin_ia32_sha1rnds4">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<ArgIndex<2>>]>; - def int_x86_sha1nexte : GCCBuiltin<"__builtin_ia32_sha1nexte">, + def int_x86_sha1nexte : ClangBuiltin<"__builtin_ia32_sha1nexte">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sha1msg1 : GCCBuiltin<"__builtin_ia32_sha1msg1">, + def int_x86_sha1msg1 : ClangBuiltin<"__builtin_ia32_sha1msg1">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sha1msg2 : GCCBuiltin<"__builtin_ia32_sha1msg2">, + def int_x86_sha1msg2 : ClangBuiltin<"__builtin_ia32_sha1msg2">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sha256rnds2 : GCCBuiltin<"__builtin_ia32_sha256rnds2">, + def int_x86_sha256rnds2 : ClangBuiltin<"__builtin_ia32_sha256rnds2">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sha256msg1 : GCCBuiltin<"__builtin_ia32_sha256msg1">, + def int_x86_sha256msg1 : ClangBuiltin<"__builtin_ia32_sha256msg1">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_sha256msg2 : GCCBuiltin<"__builtin_ia32_sha256msg2">, + def int_x86_sha256msg2 : ClangBuiltin<"__builtin_ia32_sha256msg2">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; } @@ -4806,17 +4816,17 @@ let TargetPrefix = "x86" in { // Thread synchronization ops with timer. let TargetPrefix = "x86" in { def int_x86_monitorx - : GCCBuiltin<"__builtin_ia32_monitorx">, + : ClangBuiltin<"__builtin_ia32_monitorx">, Intrinsic<[], [ llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty ], []>; def int_x86_mwaitx - : GCCBuiltin<"__builtin_ia32_mwaitx">, + : ClangBuiltin<"__builtin_ia32_mwaitx">, Intrinsic<[], [ llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], []>; } //===----------------------------------------------------------------------===// // Cache-line zero let TargetPrefix = "x86" in { - def int_x86_clzero : GCCBuiltin<"__builtin_ia32_clzero">, + def int_x86_clzero : ClangBuiltin<"__builtin_ia32_clzero">, Intrinsic<[], [llvm_ptr_ty], []>; } @@ -4825,11 +4835,11 @@ let TargetPrefix = "x86" in { let TargetPrefix = "x86" in { // Write back and invalidate - def int_x86_wbinvd : GCCBuiltin<"__builtin_ia32_wbinvd">, + def int_x86_wbinvd : ClangBuiltin<"__builtin_ia32_wbinvd">, Intrinsic<[], [], []>; // Write back no-invalidate - def int_x86_wbnoinvd : GCCBuiltin<"__builtin_ia32_wbnoinvd">, + def int_x86_wbnoinvd : ClangBuiltin<"__builtin_ia32_wbnoinvd">, Intrinsic<[], [], []>; } @@ -4837,18 +4847,18 @@ let TargetPrefix = "x86" in { // Cache-line demote let TargetPrefix = "x86" in { - def int_x86_cldemote : GCCBuiltin<"__builtin_ia32_cldemote">, + def int_x86_cldemote : ClangBuiltin<"__builtin_ia32_cldemote">, Intrinsic<[], [llvm_ptr_ty], []>; } //===----------------------------------------------------------------------===// // Wait and pause enhancements let TargetPrefix = "x86" in { - def int_x86_umonitor : GCCBuiltin<"__builtin_ia32_umonitor">, + def int_x86_umonitor : ClangBuiltin<"__builtin_ia32_umonitor">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_umwait : GCCBuiltin<"__builtin_ia32_umwait">, + def int_x86_umwait : ClangBuiltin<"__builtin_ia32_umwait">, Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; - def int_x86_tpause : GCCBuiltin<"__builtin_ia32_tpause">, + def int_x86_tpause : ClangBuiltin<"__builtin_ia32_tpause">, Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; } @@ -4856,11 +4866,11 @@ let TargetPrefix = "x86" in { // Direct Move Instructions let TargetPrefix = "x86" in { - def int_x86_directstore32 : GCCBuiltin<"__builtin_ia32_directstore_u32">, + def int_x86_directstore32 : ClangBuiltin<"__builtin_ia32_directstore_u32">, Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], []>; - def int_x86_directstore64 : GCCBuiltin<"__builtin_ia32_directstore_u64">, + def int_x86_directstore64 : ClangBuiltin<"__builtin_ia32_directstore_u64">, Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], []>; - def int_x86_movdir64b : GCCBuiltin<"__builtin_ia32_movdir64b">, + def int_x86_movdir64b : ClangBuiltin<"__builtin_ia32_movdir64b">, Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], []>; } @@ -4868,9 +4878,9 @@ let TargetPrefix = "x86" in { // PTWrite - Write data to processor trace pocket let TargetPrefix = "x86" in { - def int_x86_ptwrite32 : GCCBuiltin<"__builtin_ia32_ptwrite32">, + def int_x86_ptwrite32 : ClangBuiltin<"__builtin_ia32_ptwrite32">, Intrinsic<[], [llvm_i32_ty], []>; - def int_x86_ptwrite64 : GCCBuiltin<"__builtin_ia32_ptwrite64">, + def int_x86_ptwrite64 : ClangBuiltin<"__builtin_ia32_ptwrite64">, Intrinsic<[], [llvm_i64_ty], []>; } @@ -4878,21 +4888,21 @@ let TargetPrefix = "x86" in { // INVPCID - Invalidate Process-Context Identifier let TargetPrefix = "x86" in { - def int_x86_invpcid : GCCBuiltin<"__builtin_ia32_invpcid">, + def int_x86_invpcid : ClangBuiltin<"__builtin_ia32_invpcid">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; } let TargetPrefix = "x86" in { def int_x86_avx512bf16_cvtne2ps2bf16_128: - GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_128">, + ClangBuiltin<"__builtin_ia32_cvtne2ps2bf16_128">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; def int_x86_avx512bf16_cvtne2ps2bf16_256: - GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_256">, + ClangBuiltin<"__builtin_ia32_cvtne2ps2bf16_256">, Intrinsic<[llvm_v16i16_ty], [llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; def int_x86_avx512bf16_cvtne2ps2bf16_512: - GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_512">, + ClangBuiltin<"__builtin_ia32_cvtne2ps2bf16_512">, Intrinsic<[llvm_v32i16_ty], [llvm_v16f32_ty, llvm_v16f32_ty], [IntrNoMem]>; // Intrinsic must be masked due to it producing less than 128 bits of results. @@ -4901,21 +4911,21 @@ let TargetPrefix = "x86" in { [llvm_v4f32_ty, llvm_v8i16_ty, llvm_v4i1_ty], [IntrNoMem]>; def int_x86_avx512bf16_cvtneps2bf16_256: - GCCBuiltin<"__builtin_ia32_cvtneps2bf16_256">, + ClangBuiltin<"__builtin_ia32_cvtneps2bf16_256">, Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty], [IntrNoMem]>; def int_x86_avx512bf16_cvtneps2bf16_512: - GCCBuiltin<"__builtin_ia32_cvtneps2bf16_512">, + ClangBuiltin<"__builtin_ia32_cvtneps2bf16_512">, Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty], [IntrNoMem]>; def int_x86_avx512bf16_dpbf16ps_128: - GCCBuiltin<"__builtin_ia32_dpbf16ps_128">, + ClangBuiltin<"__builtin_ia32_dpbf16ps_128">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512bf16_dpbf16ps_256: - GCCBuiltin<"__builtin_ia32_dpbf16ps_256">, + ClangBuiltin<"__builtin_ia32_dpbf16ps_256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512bf16_dpbf16ps_512: - GCCBuiltin<"__builtin_ia32_dpbf16ps_512">, + ClangBuiltin<"__builtin_ia32_dpbf16ps_512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; } @@ -4924,9 +4934,9 @@ let TargetPrefix = "x86" in { // ENQCMD - Enqueue Stores Instructions let TargetPrefix = "x86" in { - def int_x86_enqcmd : GCCBuiltin<"__builtin_ia32_enqcmd">, + def int_x86_enqcmd : ClangBuiltin<"__builtin_ia32_enqcmd">, Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>; - def int_x86_enqcmds : GCCBuiltin<"__builtin_ia32_enqcmds">, + def int_x86_enqcmds : ClangBuiltin<"__builtin_ia32_enqcmds">, Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>; } @@ -4934,7 +4944,7 @@ let TargetPrefix = "x86" in { // SERIALIZE - Serialize instruction fetch and execution let TargetPrefix = "x86" in { - def int_x86_serialize : GCCBuiltin<"__builtin_ia32_serialize">, + def int_x86_serialize : ClangBuiltin<"__builtin_ia32_serialize">, Intrinsic<[], [], []>; } @@ -4942,16 +4952,16 @@ let TargetPrefix = "x86" in { // TSXLDTRK - TSX Suspend Load Address Tracking let TargetPrefix = "x86" in { - def int_x86_xsusldtrk : GCCBuiltin<"__builtin_ia32_xsusldtrk">, + def int_x86_xsusldtrk : ClangBuiltin<"__builtin_ia32_xsusldtrk">, Intrinsic<[], [], []>; - def int_x86_xresldtrk : GCCBuiltin<"__builtin_ia32_xresldtrk">, + def int_x86_xresldtrk : ClangBuiltin<"__builtin_ia32_xresldtrk">, Intrinsic<[], [], []>; } //===----------------------------------------------------------------------===// // Key Locker let TargetPrefix = "x86" in { - def int_x86_loadiwkey : GCCBuiltin<"__builtin_ia32_loadiwkey">, + def int_x86_loadiwkey : ClangBuiltin<"__builtin_ia32_loadiwkey">, Intrinsic<[], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], []>; def int_x86_encodekey128 : @@ -5004,91 +5014,91 @@ let TargetPrefix = "x86" in { // AMX - Intel AMX extensions let TargetPrefix = "x86" in { - def int_x86_ldtilecfg : GCCBuiltin<"__builtin_ia32_tile_loadconfig">, + def int_x86_ldtilecfg : ClangBuiltin<"__builtin_ia32_tile_loadconfig">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_sttilecfg : GCCBuiltin<"__builtin_ia32_tile_storeconfig">, + def int_x86_sttilecfg : ClangBuiltin<"__builtin_ia32_tile_storeconfig">, Intrinsic<[], [llvm_ptr_ty], []>; - def int_x86_tilerelease : GCCBuiltin<"__builtin_ia32_tilerelease">, + def int_x86_tilerelease : ClangBuiltin<"__builtin_ia32_tilerelease">, Intrinsic<[], [], []>; - def int_x86_tilezero : GCCBuiltin<"__builtin_ia32_tilezero">, + def int_x86_tilezero : ClangBuiltin<"__builtin_ia32_tilezero">, Intrinsic<[], [llvm_i8_ty], [ImmArg<ArgIndex<0>>]>; - def int_x86_tileloadd64 : GCCBuiltin<"__builtin_ia32_tileloadd64">, + def int_x86_tileloadd64 : ClangBuiltin<"__builtin_ia32_tileloadd64">, Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], [ImmArg<ArgIndex<0>>]>; - def int_x86_tileloaddt164 : GCCBuiltin<"__builtin_ia32_tileloaddt164">, + def int_x86_tileloaddt164 : ClangBuiltin<"__builtin_ia32_tileloaddt164">, Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], [ImmArg<ArgIndex<0>>]>; - def int_x86_tilestored64 : GCCBuiltin<"__builtin_ia32_tilestored64">, + def int_x86_tilestored64 : ClangBuiltin<"__builtin_ia32_tilestored64">, Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], [ImmArg<ArgIndex<0>>]>; - def int_x86_tdpbssd : GCCBuiltin<"__builtin_ia32_tdpbssd">, + def int_x86_tdpbssd : ClangBuiltin<"__builtin_ia32_tdpbssd">, Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_tdpbsud : GCCBuiltin<"__builtin_ia32_tdpbsud">, + def int_x86_tdpbsud : ClangBuiltin<"__builtin_ia32_tdpbsud">, Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_tdpbusd : GCCBuiltin<"__builtin_ia32_tdpbusd">, + def int_x86_tdpbusd : ClangBuiltin<"__builtin_ia32_tdpbusd">, Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_tdpbuud : GCCBuiltin<"__builtin_ia32_tdpbuud">, + def int_x86_tdpbuud : ClangBuiltin<"__builtin_ia32_tdpbuud">, Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_tdpbf16ps : GCCBuiltin<"__builtin_ia32_tdpbf16ps">, + def int_x86_tdpbf16ps : ClangBuiltin<"__builtin_ia32_tdpbf16ps">, Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; // AMX - internal intrinsics def int_x86_ldtilecfg_internal : - GCCBuiltin<"__builtin_ia32_tile_loadconfig_internal">, + ClangBuiltin<"__builtin_ia32_tile_loadconfig_internal">, Intrinsic<[], [llvm_ptr_ty], []>; def int_x86_tileloadd64_internal : - GCCBuiltin<"__builtin_ia32_tileloadd64_internal">, + ClangBuiltin<"__builtin_ia32_tileloadd64_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], []>; def int_x86_tileloaddt164_internal : - GCCBuiltin<"__builtin_ia32_tileloaddt164_internal">, + ClangBuiltin<"__builtin_ia32_tileloaddt164_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], []>; def int_x86_tdpbssd_internal : - GCCBuiltin<"__builtin_ia32_tdpbssd_internal">, + ClangBuiltin<"__builtin_ia32_tdpbssd_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; def int_x86_tdpbsud_internal : - GCCBuiltin<"__builtin_ia32_tdpbsud_internal">, + ClangBuiltin<"__builtin_ia32_tdpbsud_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; def int_x86_tdpbusd_internal : - GCCBuiltin<"__builtin_ia32_tdpbusd_internal">, + ClangBuiltin<"__builtin_ia32_tdpbusd_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; def int_x86_tdpbuud_internal : - GCCBuiltin<"__builtin_ia32_tdpbuud_internal">, + ClangBuiltin<"__builtin_ia32_tdpbuud_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; def int_x86_tilestored64_internal : - GCCBuiltin<"__builtin_ia32_tilestored64_internal">, + ClangBuiltin<"__builtin_ia32_tilestored64_internal">, Intrinsic<[], [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty, llvm_x86amx_ty], []>; def int_x86_tilezero_internal : - GCCBuiltin<"__builtin_ia32_tilezero_internal">, + ClangBuiltin<"__builtin_ia32_tilezero_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty], []>; def int_x86_tdpbf16ps_internal : - GCCBuiltin<"__builtin_ia32_tdpbf16ps_internal">, + ClangBuiltin<"__builtin_ia32_tdpbf16ps_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, @@ -5103,13 +5113,13 @@ let TargetPrefix = "x86" in { // UINTR - User Level Interrupt let TargetPrefix = "x86" in { - def int_x86_clui : GCCBuiltin<"__builtin_ia32_clui">, + def int_x86_clui : ClangBuiltin<"__builtin_ia32_clui">, Intrinsic<[], [], []>; - def int_x86_stui : GCCBuiltin<"__builtin_ia32_stui">, + def int_x86_stui : ClangBuiltin<"__builtin_ia32_stui">, Intrinsic<[], [], []>; - def int_x86_testui : GCCBuiltin<"__builtin_ia32_testui">, + def int_x86_testui : ClangBuiltin<"__builtin_ia32_testui">, Intrinsic<[llvm_i8_ty], [], []>; - def int_x86_senduipi : GCCBuiltin<"__builtin_ia32_senduipi">, + def int_x86_senduipi : ClangBuiltin<"__builtin_ia32_senduipi">, Intrinsic<[], [llvm_i64_ty], []>; } @@ -5117,48 +5127,48 @@ let TargetPrefix = "x86" in { // avx512_fp16: vaddph let TargetPrefix = "x86" in { def int_x86_avx512fp16_add_ph_512 - : GCCBuiltin<"__builtin_ia32_addph512">, + : ClangBuiltin<"__builtin_ia32_addph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_sub_ph_512 - : GCCBuiltin<"__builtin_ia32_subph512">, + : ClangBuiltin<"__builtin_ia32_subph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_mul_ph_512 - : GCCBuiltin<"__builtin_ia32_mulph512">, + : ClangBuiltin<"__builtin_ia32_mulph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_div_ph_512 - : GCCBuiltin<"__builtin_ia32_divph512">, + : ClangBuiltin<"__builtin_ia32_divph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_max_ph_128 - : GCCBuiltin<"__builtin_ia32_maxph128">, + : ClangBuiltin<"__builtin_ia32_maxph128">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_max_ph_256 - : GCCBuiltin<"__builtin_ia32_maxph256">, + : ClangBuiltin<"__builtin_ia32_maxph256">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_max_ph_512 - : GCCBuiltin<"__builtin_ia32_maxph512">, + : ClangBuiltin<"__builtin_ia32_maxph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_min_ph_128 - : GCCBuiltin<"__builtin_ia32_minph128">, + : ClangBuiltin<"__builtin_ia32_minph128">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_min_ph_256 - : GCCBuiltin<"__builtin_ia32_minph256">, + : ClangBuiltin<"__builtin_ia32_minph256">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_min_ph_512 - : GCCBuiltin<"__builtin_ia32_minph512">, + : ClangBuiltin<"__builtin_ia32_minph512">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; @@ -5178,367 +5188,367 @@ let TargetPrefix = "x86" in { [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_mask_add_sh_round - : GCCBuiltin<"__builtin_ia32_addsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_addsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_sub_sh_round - : GCCBuiltin<"__builtin_ia32_subsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_subsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_mul_sh_round - : GCCBuiltin<"__builtin_ia32_mulsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_mulsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_div_sh_round - : GCCBuiltin<"__builtin_ia32_divsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_divsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_min_sh_round - : GCCBuiltin<"__builtin_ia32_minsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_minsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_max_sh_round - : GCCBuiltin<"__builtin_ia32_maxsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_maxsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_cmp_sh - : GCCBuiltin<"__builtin_ia32_cmpsh_mask">, + : ClangBuiltin<"__builtin_ia32_cmpsh_mask">, Intrinsic<[ llvm_i8_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_vcomi_sh - : GCCBuiltin<"__builtin_ia32_vcomish">, + : ClangBuiltin<"__builtin_ia32_vcomish">, Intrinsic<[ llvm_i32_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtph2psx_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2psx128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2psx128_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v8f16_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2psx_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2psx256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2psx256_mask">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f16_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2psx_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2psx512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2psx512_mask">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f16_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtps2phx_128 - : GCCBuiltin<"__builtin_ia32_vcvtps2phx128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtps2phx128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4f32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtps2phx_256 - : GCCBuiltin<"__builtin_ia32_vcvtps2phx256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtps2phx256_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtps2phx_512 - : GCCBuiltin<"__builtin_ia32_vcvtps2phx512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtps2phx512_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f32_ty, llvm_v16f16_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtpd2ph_128 - : GCCBuiltin<"__builtin_ia32_vcvtpd2ph128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtpd2ph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v2f64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtpd2ph_256 - : GCCBuiltin<"__builtin_ia32_vcvtpd2ph256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtpd2ph256_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4f64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtpd2ph_512 - : GCCBuiltin<"__builtin_ia32_vcvtpd2ph512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtpd2ph512_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f64_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtph2pd_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2pd128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2pd128_mask">, Intrinsic<[ llvm_v2f64_ty ], [ llvm_v8f16_ty, llvm_v2f64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2pd_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2pd256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2pd256_mask">, Intrinsic<[ llvm_v4f64_ty ], [ llvm_v8f16_ty, llvm_v4f64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2pd_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2pd512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2pd512_mask">, Intrinsic<[ llvm_v8f64_ty ], [ llvm_v8f16_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtsh2ss_round - : GCCBuiltin<"__builtin_ia32_vcvtsh2ss_round_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2ss_round_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v8f16_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vcvtss2sh_round - : GCCBuiltin<"__builtin_ia32_vcvtss2sh_round_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtss2sh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v4f32_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vcvtsd2sh_round - : GCCBuiltin<"__builtin_ia32_vcvtsd2sh_round_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtsd2sh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v2f64_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vcvtsh2sd_round - : GCCBuiltin<"__builtin_ia32_vcvtsh2sd_round_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2sd_round_mask">, Intrinsic<[ llvm_v2f64_ty ], [ llvm_v2f64_ty, llvm_v8f16_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vcvtph2w_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2w128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2w128_mask">, Intrinsic<[ llvm_v8i16_ty ], [ llvm_v8f16_ty, llvm_v8i16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2w_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2w256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2w256_mask">, Intrinsic<[ llvm_v16i16_ty ], [ llvm_v16f16_ty, llvm_v16i16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2w_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2w512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2w512_mask">, Intrinsic<[ llvm_v32i16_ty ], [ llvm_v32f16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvttph2w_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2w128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2w128_mask">, Intrinsic<[ llvm_v8i16_ty ], [ llvm_v8f16_ty, llvm_v8i16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2w_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2w256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2w256_mask">, Intrinsic<[ llvm_v16i16_ty ], [ llvm_v16f16_ty, llvm_v16i16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2w_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2w512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2w512_mask">, Intrinsic<[ llvm_v32i16_ty ], [ llvm_v32f16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtph2uw_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2uw128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uw128_mask">, Intrinsic<[ llvm_v8i16_ty ], [ llvm_v8f16_ty, llvm_v8i16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2uw_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2uw256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uw256_mask">, Intrinsic<[ llvm_v16i16_ty ], [ llvm_v16f16_ty, llvm_v16i16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2uw_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2uw512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uw512_mask">, Intrinsic<[ llvm_v32i16_ty ], [ llvm_v32f16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvttph2uw_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2uw128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uw128_mask">, Intrinsic<[ llvm_v8i16_ty ], [ llvm_v8f16_ty, llvm_v8i16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2uw_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2uw256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uw256_mask">, Intrinsic<[ llvm_v16i16_ty ], [ llvm_v16f16_ty, llvm_v16i16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2uw_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2uw512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uw512_mask">, Intrinsic<[ llvm_v32i16_ty ], [ llvm_v32f16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtph2dq_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2dq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2dq128_mask">, Intrinsic<[ llvm_v4i32_ty ], [ llvm_v8f16_ty, llvm_v4i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2dq_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2dq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2dq256_mask">, Intrinsic<[ llvm_v8i32_ty ], [ llvm_v8f16_ty, llvm_v8i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2dq_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2dq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2dq512_mask">, Intrinsic<[ llvm_v16i32_ty ], [ llvm_v16f16_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtph2udq_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2udq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2udq128_mask">, Intrinsic<[ llvm_v4i32_ty ], [ llvm_v8f16_ty, llvm_v4i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2udq_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2udq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2udq256_mask">, Intrinsic<[ llvm_v8i32_ty ], [ llvm_v8f16_ty, llvm_v8i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2udq_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2udq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2udq512_mask">, Intrinsic<[ llvm_v16i32_ty ], [ llvm_v16f16_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtdq2ph_128 - : GCCBuiltin<"__builtin_ia32_vcvtdq2ph128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtdq2ph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4i32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtudq2ph_128 - : GCCBuiltin<"__builtin_ia32_vcvtudq2ph128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtudq2ph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4i32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2dq_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2dq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2dq128_mask">, Intrinsic<[ llvm_v4i32_ty ], [ llvm_v8f16_ty, llvm_v4i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2dq_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2dq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2dq256_mask">, Intrinsic<[ llvm_v8i32_ty ], [ llvm_v8f16_ty, llvm_v8i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2dq_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2dq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2dq512_mask">, Intrinsic<[ llvm_v16i32_ty ], [ llvm_v16f16_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvttph2udq_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2udq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2udq128_mask">, Intrinsic<[ llvm_v4i32_ty ], [ llvm_v8f16_ty, llvm_v4i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2udq_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2udq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2udq256_mask">, Intrinsic<[ llvm_v8i32_ty ], [ llvm_v8f16_ty, llvm_v8i32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2udq_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2udq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2udq512_mask">, Intrinsic<[ llvm_v16i32_ty ], [ llvm_v16f16_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtqq2ph_128 - : GCCBuiltin<"__builtin_ia32_vcvtqq2ph128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtqq2ph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v2i64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtqq2ph_256 - : GCCBuiltin<"__builtin_ia32_vcvtqq2ph256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtqq2ph256_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4i64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2qq_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2qq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2qq128_mask">, Intrinsic<[ llvm_v2i64_ty ], [ llvm_v8f16_ty, llvm_v2i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2qq_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2qq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2qq256_mask">, Intrinsic<[ llvm_v4i64_ty ], [ llvm_v8f16_ty, llvm_v4i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2qq_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2qq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2qq512_mask">, Intrinsic<[ llvm_v8i64_ty ], [ llvm_v8f16_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvtuqq2ph_128 - : GCCBuiltin<"__builtin_ia32_vcvtuqq2ph128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtuqq2ph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v2i64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtuqq2ph_256 - : GCCBuiltin<"__builtin_ia32_vcvtuqq2ph256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtuqq2ph256_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v4i64_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2uqq_128 - : GCCBuiltin<"__builtin_ia32_vcvtph2uqq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uqq128_mask">, Intrinsic<[ llvm_v2i64_ty ], [ llvm_v8f16_ty, llvm_v2i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2uqq_256 - : GCCBuiltin<"__builtin_ia32_vcvtph2uqq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uqq256_mask">, Intrinsic<[ llvm_v4i64_ty ], [ llvm_v8f16_ty, llvm_v4i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvtph2uqq_512 - : GCCBuiltin<"__builtin_ia32_vcvtph2uqq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvtph2uqq512_mask">, Intrinsic<[ llvm_v8i64_ty ], [ llvm_v8f16_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvttph2qq_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2qq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2qq128_mask">, Intrinsic<[ llvm_v2i64_ty ], [ llvm_v8f16_ty, llvm_v2i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2qq_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2qq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2qq256_mask">, Intrinsic<[ llvm_v4i64_ty ], [ llvm_v8f16_ty, llvm_v4i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2qq_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2qq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2qq512_mask">, Intrinsic<[ llvm_v8i64_ty ], [ llvm_v8f16_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vcvttph2uqq_128 - : GCCBuiltin<"__builtin_ia32_vcvttph2uqq128_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uqq128_mask">, Intrinsic<[ llvm_v2i64_ty ], [ llvm_v8f16_ty, llvm_v2i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2uqq_256 - : GCCBuiltin<"__builtin_ia32_vcvttph2uqq256_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uqq256_mask">, Intrinsic<[ llvm_v4i64_ty ], [ llvm_v8f16_ty, llvm_v4i64_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vcvttph2uqq_512 - : GCCBuiltin<"__builtin_ia32_vcvttph2uqq512_mask">, + : ClangBuiltin<"__builtin_ia32_vcvttph2uqq512_mask">, Intrinsic<[ llvm_v8i64_ty ], [ llvm_v8f16_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_vcvtsh2si32 - : GCCBuiltin<"__builtin_ia32_vcvtsh2si32">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2si32">, Intrinsic<[ llvm_i32_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvtsh2usi32 - : GCCBuiltin<"__builtin_ia32_vcvtsh2usi32">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2usi32">, Intrinsic<[ llvm_i32_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvtsh2si64 - : GCCBuiltin<"__builtin_ia32_vcvtsh2si64">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2si64">, Intrinsic<[ llvm_i64_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvtsh2usi64 - : GCCBuiltin<"__builtin_ia32_vcvtsh2usi64">, + : ClangBuiltin<"__builtin_ia32_vcvtsh2usi64">, Intrinsic<[ llvm_i64_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvtusi2sh - : GCCBuiltin<"__builtin_ia32_vcvtusi2sh">, + : ClangBuiltin<"__builtin_ia32_vcvtusi2sh">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_vcvtusi642sh - : GCCBuiltin<"__builtin_ia32_vcvtusi642sh">, + : ClangBuiltin<"__builtin_ia32_vcvtusi642sh">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i64_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_vcvtsi2sh - : GCCBuiltin<"__builtin_ia32_vcvtsi2sh">, + : ClangBuiltin<"__builtin_ia32_vcvtsi2sh">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_vcvtsi642sh - : GCCBuiltin<"__builtin_ia32_vcvtsi642sh">, + : ClangBuiltin<"__builtin_ia32_vcvtsi642sh">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i64_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>> ]>; def int_x86_avx512fp16_vcvttsh2si32 - : GCCBuiltin<"__builtin_ia32_vcvttsh2si32">, + : ClangBuiltin<"__builtin_ia32_vcvttsh2si32">, Intrinsic<[ llvm_i32_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvttsh2si64 - : GCCBuiltin<"__builtin_ia32_vcvttsh2si64">, + : ClangBuiltin<"__builtin_ia32_vcvttsh2si64">, Intrinsic<[ llvm_i64_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvttsh2usi32 - : GCCBuiltin<"__builtin_ia32_vcvttsh2usi32">, + : ClangBuiltin<"__builtin_ia32_vcvttsh2usi32">, Intrinsic<[ llvm_i32_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_vcvttsh2usi64 - : GCCBuiltin<"__builtin_ia32_vcvttsh2usi64">, + : ClangBuiltin<"__builtin_ia32_vcvttsh2usi64">, Intrinsic<[ llvm_i64_ty ], [ llvm_v8f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; @@ -5551,61 +5561,61 @@ let TargetPrefix = "x86" in { llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_rsqrt_ph_128 - : GCCBuiltin<"__builtin_ia32_rsqrtph128_mask">, + : ClangBuiltin<"__builtin_ia32_rsqrtph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rsqrt_ph_256 - : GCCBuiltin<"__builtin_ia32_rsqrtph256_mask">, + : ClangBuiltin<"__builtin_ia32_rsqrtph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rsqrt_ph_512 - : GCCBuiltin<"__builtin_ia32_rsqrtph512_mask">, + : ClangBuiltin<"__builtin_ia32_rsqrtph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rsqrt_sh - : GCCBuiltin<"__builtin_ia32_rsqrtsh_mask">, + : ClangBuiltin<"__builtin_ia32_rsqrtsh_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rcp_ph_128 - : GCCBuiltin<"__builtin_ia32_rcpph128_mask">, + : ClangBuiltin<"__builtin_ia32_rcpph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rcp_ph_256 - : GCCBuiltin<"__builtin_ia32_rcpph256_mask">, + : ClangBuiltin<"__builtin_ia32_rcpph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rcp_ph_512 - : GCCBuiltin<"__builtin_ia32_rcpph512_mask">, + : ClangBuiltin<"__builtin_ia32_rcpph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_rcp_sh - : GCCBuiltin<"__builtin_ia32_rcpsh_mask">, + : ClangBuiltin<"__builtin_ia32_rcpsh_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_reduce_ph_128 - : GCCBuiltin<"__builtin_ia32_reduceph128_mask">, + : ClangBuiltin<"__builtin_ia32_reduceph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_reduce_ph_256 - : GCCBuiltin<"__builtin_ia32_reduceph256_mask">, + : ClangBuiltin<"__builtin_ia32_reduceph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_i32_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_reduce_ph_512 - : GCCBuiltin<"__builtin_ia32_reduceph512_mask">, + : ClangBuiltin<"__builtin_ia32_reduceph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_i32_ty, llvm_v32f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_reduce_sh - : GCCBuiltin<"__builtin_ia32_reducesh_mask">, + : ClangBuiltin<"__builtin_ia32_reducesh_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty ], @@ -5620,91 +5630,91 @@ let TargetPrefix = "x86" in { : Intrinsic<[ llvm_v32i1_ty ], [ llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_fpclass_sh - : GCCBuiltin<"__builtin_ia32_fpclasssh_mask">, + : ClangBuiltin<"__builtin_ia32_fpclasssh_mask">, Intrinsic<[ llvm_i8_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_i8_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_getexp_ph_128 - : GCCBuiltin<"__builtin_ia32_getexpph128_mask">, + : ClangBuiltin<"__builtin_ia32_getexpph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_getexp_ph_256 - : GCCBuiltin<"__builtin_ia32_getexpph256_mask">, + : ClangBuiltin<"__builtin_ia32_getexpph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_getexp_ph_512 - : GCCBuiltin<"__builtin_ia32_getexpph512_mask">, + : ClangBuiltin<"__builtin_ia32_getexpph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_getexp_sh - : GCCBuiltin<"__builtin_ia32_getexpsh128_round_mask">, + : ClangBuiltin<"__builtin_ia32_getexpsh128_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_getmant_ph_128 - : GCCBuiltin<"__builtin_ia32_getmantph128_mask">, + : ClangBuiltin<"__builtin_ia32_getmantph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_getmant_ph_256 - : GCCBuiltin<"__builtin_ia32_getmantph256_mask">, + : ClangBuiltin<"__builtin_ia32_getmantph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_i32_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_getmant_ph_512 - : GCCBuiltin<"__builtin_ia32_getmantph512_mask">, + : ClangBuiltin<"__builtin_ia32_getmantph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_i32_ty, llvm_v32f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_getmant_sh - : GCCBuiltin<"__builtin_ia32_getmantsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_getmantsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_i32_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<5>> ]>; def int_x86_avx512fp16_mask_rndscale_ph_128 - : GCCBuiltin<"__builtin_ia32_rndscaleph_128_mask">, + : ClangBuiltin<"__builtin_ia32_rndscaleph_128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_i32_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_rndscale_ph_256 - : GCCBuiltin<"__builtin_ia32_rndscaleph_256_mask">, + : ClangBuiltin<"__builtin_ia32_rndscaleph_256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_i32_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>> ]>; def int_x86_avx512fp16_mask_rndscale_ph_512 - : GCCBuiltin<"__builtin_ia32_rndscaleph_mask">, + : ClangBuiltin<"__builtin_ia32_rndscaleph_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_i32_ty, llvm_v32f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_rndscale_sh - : GCCBuiltin<"__builtin_ia32_rndscalesh_round_mask">, + : ClangBuiltin<"__builtin_ia32_rndscalesh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>> ]>; def int_x86_avx512fp16_mask_scalef_ph_128 - : GCCBuiltin<"__builtin_ia32_scalefph128_mask">, + : ClangBuiltin<"__builtin_ia32_scalefph128_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_scalef_ph_256 - : GCCBuiltin<"__builtin_ia32_scalefph256_mask">, + : ClangBuiltin<"__builtin_ia32_scalefph256_mask">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty, llvm_v16f16_ty, llvm_i16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_scalef_ph_512 - : GCCBuiltin<"__builtin_ia32_scalefph512_mask">, + : ClangBuiltin<"__builtin_ia32_scalefph512_mask">, Intrinsic<[ llvm_v32f16_ty ], [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_scalef_sh - : GCCBuiltin<"__builtin_ia32_scalefsh_round_mask">, + : ClangBuiltin<"__builtin_ia32_scalefsh_round_mask">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty, llvm_i8_ty, llvm_i32_ty ], @@ -5715,12 +5725,12 @@ let TargetPrefix = "x86" in { [ llvm_v32f16_ty, llvm_v32f16_ty, llvm_v32f16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_vfmaddsub_ph_128 - : GCCBuiltin<"__builtin_ia32_vfmaddsubph">, + : ClangBuiltin<"__builtin_ia32_vfmaddsubph">, Intrinsic<[ llvm_v8f16_ty ], [ llvm_v8f16_ty, llvm_v8f16_ty, llvm_v8f16_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_vfmaddsub_ph_256 - : GCCBuiltin<"__builtin_ia32_vfmaddsubph256">, + : ClangBuiltin<"__builtin_ia32_vfmaddsubph256">, Intrinsic<[ llvm_v16f16_ty ], [ llvm_v16f16_ty, llvm_v16f16_ty, llvm_v16f16_ty ], [ IntrNoMem ]>; @@ -5734,133 +5744,133 @@ let TargetPrefix = "x86" in { [ IntrNoMem, ImmArg<ArgIndex<3>> ]>; def int_x86_avx512fp16_mask_vfcmadd_cph_128 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph128_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph128_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_maskz_vfcmadd_cph_128 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph128_maskz">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph128_maskz">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfcmadd_cph_256 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph256_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph256_mask">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_maskz_vfcmadd_cph_256 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph256_maskz">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph256_maskz">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfcmadd_cph_512 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph512_mask3">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph512_mask3">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_maskz_vfcmadd_cph_512 - : GCCBuiltin<"__builtin_ia32_vfcmaddcph512_maskz">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcph512_maskz">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfmadd_cph_128 - : GCCBuiltin<"__builtin_ia32_vfmaddcph128_mask">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph128_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_maskz_vfmadd_cph_128 - : GCCBuiltin<"__builtin_ia32_vfmaddcph128_maskz">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph128_maskz">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfmadd_cph_256 - : GCCBuiltin<"__builtin_ia32_vfmaddcph256_mask">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph256_mask">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_maskz_vfmadd_cph_256 - : GCCBuiltin<"__builtin_ia32_vfmaddcph256_maskz">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph256_maskz">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfmadd_cph_512 - : GCCBuiltin<"__builtin_ia32_vfmaddcph512_mask3">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph512_mask3">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_maskz_vfmadd_cph_512 - : GCCBuiltin<"__builtin_ia32_vfmaddcph512_maskz">, + : ClangBuiltin<"__builtin_ia32_vfmaddcph512_maskz">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfmadd_csh - : GCCBuiltin<"__builtin_ia32_vfmaddcsh_mask">, + : ClangBuiltin<"__builtin_ia32_vfmaddcsh_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_maskz_vfmadd_csh - : GCCBuiltin<"__builtin_ia32_vfmaddcsh_maskz">, + : ClangBuiltin<"__builtin_ia32_vfmaddcsh_maskz">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfcmadd_csh - : GCCBuiltin<"__builtin_ia32_vfcmaddcsh_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcsh_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_maskz_vfcmadd_csh - : GCCBuiltin<"__builtin_ia32_vfcmaddcsh_maskz">, + : ClangBuiltin<"__builtin_ia32_vfcmaddcsh_maskz">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfmul_cph_128 - : GCCBuiltin<"__builtin_ia32_vfmulcph128_mask">, + : ClangBuiltin<"__builtin_ia32_vfmulcph128_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfcmul_cph_128 - : GCCBuiltin<"__builtin_ia32_vfcmulcph128_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmulcph128_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfmul_cph_256 - : GCCBuiltin<"__builtin_ia32_vfmulcph256_mask">, + : ClangBuiltin<"__builtin_ia32_vfmulcph256_mask">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfcmul_cph_256 - : GCCBuiltin<"__builtin_ia32_vfcmulcph256_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmulcph256_mask">, Intrinsic<[ llvm_v8f32_ty ], [ llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty ], [ IntrNoMem ]>; def int_x86_avx512fp16_mask_vfmul_cph_512 - : GCCBuiltin<"__builtin_ia32_vfmulcph512_mask">, + : ClangBuiltin<"__builtin_ia32_vfmulcph512_mask">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfcmul_cph_512 - : GCCBuiltin<"__builtin_ia32_vfcmulcph512_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmulcph512_mask">, Intrinsic<[ llvm_v16f32_ty ], [ llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfmul_csh - : GCCBuiltin<"__builtin_ia32_vfmulcsh_mask">, + : ClangBuiltin<"__builtin_ia32_vfmulcsh_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], [ IntrNoMem, ImmArg<ArgIndex<4>> ]>; def int_x86_avx512fp16_mask_vfcmul_csh - : GCCBuiltin<"__builtin_ia32_vfcmulcsh_mask">, + : ClangBuiltin<"__builtin_ia32_vfcmulcsh_mask">, Intrinsic<[ llvm_v4f32_ty ], [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty ], diff --git a/llvm/include/llvm/IR/IntrinsicsXCore.td b/llvm/include/llvm/IR/IntrinsicsXCore.td index 89dbc65fea44..d2afc3497833 100644 --- a/llvm/include/llvm/IR/IntrinsicsXCore.td +++ b/llvm/include/llvm/IR/IntrinsicsXCore.td @@ -13,7 +13,7 @@ let TargetPrefix = "xcore" in { // All intrinsics start with "llvm.xcore.". // Miscellaneous instructions. def int_xcore_bitrev : Intrinsic<[llvm_i32_ty],[llvm_i32_ty],[IntrNoMem]>, - GCCBuiltin<"__builtin_bitrev">; + ClangBuiltin<"__builtin_bitrev">; def int_xcore_crc8 : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_i32_ty,llvm_i32_ty,llvm_i32_ty], [IntrNoMem]>; @@ -25,11 +25,11 @@ let TargetPrefix = "xcore" in { // All intrinsics start with "llvm.xcore.". def int_xcore_zext : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_xcore_getid : Intrinsic<[llvm_i32_ty],[],[IntrNoMem]>, - GCCBuiltin<"__builtin_getid">; + ClangBuiltin<"__builtin_getid">; def int_xcore_getps : Intrinsic<[llvm_i32_ty],[llvm_i32_ty]>, - GCCBuiltin<"__builtin_getps">; + ClangBuiltin<"__builtin_getps">; def int_xcore_setps : Intrinsic<[],[llvm_i32_ty, llvm_i32_ty]>, - GCCBuiltin<"__builtin_setps">; + ClangBuiltin<"__builtin_setps">; def int_xcore_geted : Intrinsic<[llvm_i32_ty],[]>; def int_xcore_getet : Intrinsic<[llvm_i32_ty],[]>; def int_xcore_setsr : Intrinsic<[],[llvm_i32_ty]>; diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index 446bcecf1c64..91712df153a0 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -24,6 +24,7 @@ namespace llvm { +class Any; class DiagnosticInfo; enum DiagnosticSeverity : char; class Function; @@ -93,6 +94,7 @@ public: OB_preallocated = 4, // "preallocated" OB_gc_live = 5, // "gc-live" OB_clang_arc_attachedcall = 6, // "clang.arc.attachedcall" + OB_ptrauth = 7, // "ptrauth" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. @@ -201,6 +203,11 @@ public: /// diagnostics. void setDiagnosticsHotnessRequested(bool Requested); + bool getMisExpectWarningRequested() const; + void setMisExpectWarningRequested(bool Requested); + void setDiagnosticsMisExpectTolerance(Optional<uint64_t> Tolerance); + uint64_t getDiagnosticsMisExpectTolerance() const; + /// Return the minimum hotness value a diagnostic would need in order /// to be included in optimization diagnostics. /// @@ -304,13 +311,22 @@ public: /// LLVMContext is used by compilation. void setOptPassGate(OptPassGate&); - /// Enable opaque pointers. Can only be called before creating the first - /// pointer type. - void enableOpaquePointers() const; + /// Whether we've decided on using opaque pointers or typed pointers yet. + bool hasSetOpaquePointersValue() const; + + /// Set whether opaque pointers are enabled. The method may be called multiple + /// times, but only with the same value. Note that creating a pointer type or + /// otherwise querying the opaque pointer mode performs an implicit set to + /// the default value. + void setOpaquePointers(bool Enable) const; /// Whether typed pointers are supported. If false, all pointers are opaque. bool supportsTypedPointers() const; + /// Optionally target-spcific data can be attached to the context for lifetime + /// management and bypassing layering restrictions. + llvm::Any &getTargetData() const; + private: // Module needs access to the add/removeModule methods. friend class Module; diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h index 311a407f1a19..41c11d26aa45 100644 --- a/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/llvm/include/llvm/IR/LegacyPassManagers.h @@ -294,9 +294,7 @@ private: /// used by pass managers. class PMDataManager { public: - explicit PMDataManager() : TPM(nullptr), Depth(0) { - initializeAnalysisInfo(); - } + explicit PMDataManager() { initializeAnalysisInfo(); } virtual ~PMDataManager(); @@ -418,7 +416,7 @@ public: protected: // Top level manager. - PMTopLevelManager *TPM; + PMTopLevelManager *TPM = nullptr; // Collection of pass that are managed by this manager SmallVector<Pass *, 16> PassVector; @@ -446,7 +444,7 @@ private: // this manager. SmallVector<Pass *, 16> HigherLevelAnalysis; - unsigned Depth; + unsigned Depth = 0; }; //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index 42829388b79a..21d7b8b6da71 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -108,6 +108,10 @@ public: /// Merge the new callback encoding \p NewCB into \p ExistingCallbacks. MDNode *mergeCallbackEncodings(MDNode *ExistingCallbacks, MDNode *NewCB); + /// Return metadata feeding to the CodeGen about how to generate a function + /// prologue for the "function" santizier. + MDNode *createRTTIPointerPrologue(Constant *PrologueSig, Constant *RTTI); + //===------------------------------------------------------------------===// // AA metadata. //===------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/MatrixBuilder.h b/llvm/include/llvm/IR/MatrixBuilder.h index 4c8286692ebf..dbf2cfb7c5e9 100644 --- a/llvm/include/llvm/IR/MatrixBuilder.h +++ b/llvm/include/llvm/IR/MatrixBuilder.h @@ -30,8 +30,8 @@ class Function; class Twine; class Module; -template <class IRBuilderTy> class MatrixBuilder { - IRBuilderTy &B; +class MatrixBuilder { + IRBuilderBase &B; Module *getModule() { return B.GetInsertBlock()->getParent()->getParent(); } std::pair<Value *, Value *> splatScalarOperandIfNeeded(Value *LHS, @@ -55,21 +55,17 @@ template <class IRBuilderTy> class MatrixBuilder { } public: - MatrixBuilder(IRBuilderTy &Builder) : B(Builder) {} + MatrixBuilder(IRBuilderBase &Builder) : B(Builder) {} /// Create a column major, strided matrix load. + /// \p EltTy - Matrix element type /// \p DataPtr - Start address of the matrix read /// \p Rows - Number of rows in matrix (must be a constant) /// \p Columns - Number of columns in matrix (must be a constant) /// \p Stride - Space between columns - CallInst *CreateColumnMajorLoad(Value *DataPtr, Align Alignment, + CallInst *CreateColumnMajorLoad(Type *EltTy, Value *DataPtr, Align Alignment, Value *Stride, bool IsVolatile, unsigned Rows, unsigned Columns, const Twine &Name = "") { - - // Deal with the pointer - PointerType *PtrTy = cast<PointerType>(DataPtr->getType()); - Type *EltTy = PtrTy->getPointerElementType(); - auto *RetType = FixedVectorType::get(EltTy, Rows * Columns); Value *Ops[] = {DataPtr, Stride, B.getInt1(IsVolatile), B.getInt32(Rows), @@ -234,12 +230,11 @@ public: /// Create an assumption that \p Idx is less than \p NumElements. void CreateIndexAssumption(Value *Idx, unsigned NumElements, Twine const &Name = "") { - Value *NumElts = B.getIntN(Idx->getType()->getScalarSizeInBits(), NumElements); auto *Cmp = B.CreateICmpULT(Idx, NumElts); - if (auto *ConstCond = dyn_cast<ConstantInt>(Cmp)) - assert(ConstCond->isOne() && "Index must be valid!"); + if (isa<ConstantInt>(Cmp)) + assert(cast<ConstantInt>(Cmp)->isOne() && "Index must be valid!"); else B.CreateAssumption(Cmp); } @@ -248,7 +243,6 @@ public: /// a matrix with \p NumRows embedded in a vector. Value *CreateIndex(Value *RowIdx, Value *ColumnIdx, unsigned NumRows, Twine const &Name = "") { - unsigned MaxWidth = std::max(RowIdx->getType()->getScalarSizeInBits(), ColumnIdx->getType()->getScalarSizeInBits()); Type *IntTy = IntegerType::get(RowIdx->getType()->getContext(), MaxWidth); diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index 7965884990e5..be359d94f812 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -169,7 +169,7 @@ inline raw_ostream &operator<<(raw_ostream &OS, const Metadata &MD) { /// Metadata wrapper in the Value hierarchy. /// /// A member of the \a Value hierarchy to represent a reference to metadata. -/// This allows, e.g., instrinsics to have metadata as operands. +/// This allows, e.g., intrinsics to have metadata as operands. /// /// Notably, this is the only thing in either hierarchy that is allowed to /// reference \a LocalAsMetadata. @@ -302,7 +302,8 @@ public: /// /// Replace all uses of this with \c MD, which is allowed to be null. void replaceAllUsesWith(Metadata *MD); - + /// Replace all uses of the constant with Undef in debug info metadata + static void SalvageDebugInfo(const Constant &C); /// Returns the list of all DIArgList users of this. SmallVector<Metadata *> getAllArgListUsers(); @@ -774,10 +775,21 @@ class MDOperand { public: MDOperand() = default; - MDOperand(MDOperand &&) = delete; MDOperand(const MDOperand &) = delete; - MDOperand &operator=(MDOperand &&) = delete; + MDOperand(MDOperand &&Op) { + MD = Op.MD; + if (MD) + (void)MetadataTracking::retrack(Op.MD, MD); + Op.MD = nullptr; + } MDOperand &operator=(const MDOperand &) = delete; + MDOperand &operator=(MDOperand &&Op) { + MD = Op.MD; + if (MD) + (void)MetadataTracking::retrack(Op.MD, MD); + Op.MD = nullptr; + return *this; + } ~MDOperand() { untrack(); } Metadata *get() const { return MD; } @@ -922,13 +934,109 @@ struct TempMDNodeDeleter { /// If an unresolved node is part of a cycle, \a resolveCycles() needs /// to be called on some member of the cycle once all temporary nodes have been /// replaced. +/// +/// MDNodes can be large or small, as well as resizable or non-resizable. +/// Large MDNodes' operands are allocated in a separate storage vector, +/// whereas small MDNodes' operands are co-allocated. Distinct and temporary +/// MDnodes are resizable, but only MDTuples support this capability. +/// +/// Clients can add operands to resizable MDNodes using push_back(). class MDNode : public Metadata { friend class ReplaceableMetadataImpl; friend class LLVMContextImpl; friend class DIArgList; - unsigned NumOperands; - unsigned NumUnresolved; + /// The header that is coallocated with an MDNode along with its "small" + /// operands. It is located immediately before the main body of the node. + /// The operands are in turn located immediately before the header. + /// For resizable MDNodes, the space for the storage vector is also allocated + /// immediately before the header, overlapping with the operands. + struct Header { + bool IsResizable : 1; + bool IsLarge : 1; + size_t SmallSize : 4; + size_t SmallNumOps : 4; + size_t : sizeof(size_t) * CHAR_BIT - 10; + + unsigned NumUnresolved = 0; + using LargeStorageVector = SmallVector<MDOperand, 0>; + + static constexpr size_t NumOpsFitInVector = + sizeof(LargeStorageVector) / sizeof(MDOperand); + static_assert( + NumOpsFitInVector * sizeof(MDOperand) == sizeof(LargeStorageVector), + "sizeof(LargeStorageVector) must be a multiple of sizeof(MDOperand)"); + + static constexpr size_t MaxSmallSize = 15; + + static constexpr size_t getOpSize(unsigned NumOps) { + return sizeof(MDOperand) * NumOps; + } + /// Returns the number of operands the node has space for based on its + /// allocation characteristics. + static size_t getSmallSize(size_t NumOps, bool IsResizable, bool IsLarge) { + return IsLarge ? NumOpsFitInVector + : std::max(NumOps, NumOpsFitInVector * IsResizable); + } + /// Returns the number of bytes allocated for operands and header. + static size_t getAllocSize(StorageType Storage, size_t NumOps) { + return getOpSize( + getSmallSize(NumOps, isResizable(Storage), isLarge(NumOps))) + + sizeof(Header); + } + + /// Only temporary and distinct nodes are resizable. + static bool isResizable(StorageType Storage) { return Storage != Uniqued; } + static bool isLarge(size_t NumOps) { return NumOps > MaxSmallSize; } + + size_t getAllocSize() const { + return getOpSize(SmallSize) + sizeof(Header); + } + void *getAllocation() { + return reinterpret_cast<char *>(this + 1) - + alignTo(getAllocSize(), alignof(uint64_t)); + } + + void *getLargePtr() const; + void *getSmallPtr(); + + LargeStorageVector &getLarge() { + assert(IsLarge); + return *reinterpret_cast<LargeStorageVector *>(getLargePtr()); + } + + const LargeStorageVector &getLarge() const { + assert(IsLarge); + return *reinterpret_cast<const LargeStorageVector *>(getLargePtr()); + } + + void resizeSmall(size_t NumOps); + void resizeSmallToLarge(size_t NumOps); + void resize(size_t NumOps); + + explicit Header(size_t NumOps, StorageType Storage); + ~Header(); + + MutableArrayRef<MDOperand> operands() { + if (IsLarge) + return getLarge(); + return makeMutableArrayRef( + reinterpret_cast<MDOperand *>(this) - SmallSize, SmallNumOps); + } + + ArrayRef<MDOperand> operands() const { + if (IsLarge) + return getLarge(); + return makeArrayRef(reinterpret_cast<const MDOperand *>(this) - SmallSize, + SmallNumOps); + } + }; + + Header &getHeader() { return *(reinterpret_cast<Header *>(this) - 1); } + + const Header &getHeader() const { + return *(reinterpret_cast<const Header *>(this) - 1); + } ContextAndReplaceableUses Context; @@ -937,7 +1045,7 @@ protected: ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = None); ~MDNode() = default; - void *operator new(size_t Size, unsigned NumOps); + void *operator new(size_t Size, size_t NumOps, StorageType Storage); void operator delete(void *Mem); /// Required by std, but never called. @@ -952,8 +1060,8 @@ protected: void dropAllReferences(); - MDOperand *mutable_begin() { return mutable_end() - NumOperands; } - MDOperand *mutable_end() { return reinterpret_cast<MDOperand *>(this); } + MDOperand *mutable_begin() { return getHeader().operands().begin(); } + MDOperand *mutable_end() { return getHeader().operands().end(); } using mutable_op_range = iterator_range<MDOperand *>; @@ -999,7 +1107,7 @@ public: /// As forward declarations are resolved, their containers should get /// resolved automatically. However, if this (or one of its operands) is /// involved in a cycle, \a resolveCycles() needs to be called explicitly. - bool isResolved() const { return !isTemporary() && !NumUnresolved; } + bool isResolved() const { return !isTemporary() && !getNumUnresolved(); } bool isUniqued() const { return Storage == Uniqued; } bool isDistinct() const { return Storage == Distinct; } @@ -1093,11 +1201,25 @@ protected: /// Sets the operand directly, without worrying about uniquing. void setOperand(unsigned I, Metadata *New); + unsigned getNumUnresolved() const { return getHeader().NumUnresolved; } + + void setNumUnresolved(unsigned N) { getHeader().NumUnresolved = N; } void storeDistinctInContext(); template <class T, class StoreT> static T *storeImpl(T *N, StorageType Storage, StoreT &Store); template <class T> static T *storeImpl(T *N, StorageType Storage); + /// Resize the node to hold \a NumOps operands. + /// + /// \pre \a isTemporary() or \a isDistinct() + /// \pre MetadataID == MDTupleKind + void resize(size_t NumOps) { + assert(!isUniqued() && "Resizing is not supported for uniqued nodes"); + assert(getMetadataID() == MDTupleKind && + "Resizing is not supported for this node kind"); + getHeader().resize(NumOps); + } + private: void handleChangedOperand(void *Ref, Metadata *New); @@ -1154,12 +1276,12 @@ public: op_range operands() const { return op_range(op_begin(), op_end()); } const MDOperand &getOperand(unsigned I) const { - assert(I < NumOperands && "Out of range"); - return op_begin()[I]; + assert(I < getNumOperands() && "Out of range"); + return getHeader().operands()[I]; } /// Return number of MDNode operands. - unsigned getNumOperands() const { return NumOperands; } + unsigned getNumOperands() const { return getHeader().operands().size(); } /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Metadata *MD) { @@ -1244,6 +1366,16 @@ public: /// Return a (temporary) clone of this. TempMDTuple clone() const { return cloneImpl(); } + /// Append an element to the tuple. This will resize the node. + void push_back(Metadata *MD) { + size_t NumOps = getNumOperands(); + resize(NumOps + 1); + setOperand(NumOps, MD); + } + + /// Shrink the operands by 1. + void pop_back() { resize(getNumOperands() - 1); } + static bool classof(const Metadata *MD) { return MD->getMetadataID() == MDTupleKind; } diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h index 7b834fbeeebf..fc2d60947118 100644 --- a/llvm/include/llvm/IR/Module.h +++ b/llvm/include/llvm/IR/Module.h @@ -58,9 +58,9 @@ class VersionTuple; /// other modules) this module depends on, a symbol table, and various data /// about the target's characteristics. /// -/// A module maintains a GlobalValRefMap object that is used to hold all +/// A module maintains a GlobalList object that is used to hold all /// constant references to global variables in the module. When a global -/// variable is destroyed, it should have no entries in the GlobalValueRefMap. +/// variable is destroyed, it should have no entries in the GlobalList. /// The main container class for the LLVM Intermediate Representation. class LLVM_EXTERNAL_VISIBILITY Module { /// @name Types And Enumerations @@ -146,9 +146,12 @@ public: /// Takes the max of the two values, which are required to be integers. Max = 7, + /// Takes the min of the two values, which are required to be integers. + Min = 8, + // Markers: ModFlagBehaviorFirstVal = Error, - ModFlagBehaviorLastVal = Max + ModFlagBehaviorLastVal = Min }; /// Checks if Metadata represents a valid ModFlagBehavior, and stores the @@ -360,6 +363,8 @@ public: /// In all cases, the returned value is a FunctionCallee wrapper around the /// 'FunctionType *T' passed in, as well as a 'Value*' either of the Function or /// the bitcast to the function. + /// + /// Note: For library calls getOrInsertLibFunc() should be used instead. FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList); @@ -888,8 +893,8 @@ public: void setRtLibUseGOT(); /// Get/set whether synthesized functions should get the uwtable attribute. - bool getUwtable() const; - void setUwtable(); + UWTableKind getUwtable() const; + void setUwtable(UWTableKind Kind); /// Get/set whether synthesized functions should get the "frame-pointer" /// attribute. @@ -939,10 +944,17 @@ public: /// @returns a string containing the target variant triple. StringRef getDarwinTargetVariantTriple() const; + /// Set the target variant triple which is a string describing a variant of + /// the target host platform. + void setDarwinTargetVariantTriple(StringRef T); + /// Get the target variant version build SDK version metadata. /// /// An empty version is returned if no such metadata is attached. VersionTuple getDarwinTargetVariantSDKVersion() const; + + /// Set the target variant version build SDK version metadata. + void setDarwinTargetVariantSDKVersion(VersionTuple Version); }; /// Given "llvm.used" or "llvm.compiler.used" as a global name, collect the diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h index ec149747e3f4..4e9f772dfdb6 100644 --- a/llvm/include/llvm/IR/NoFolder.h +++ b/llvm/include/llvm/IR/NoFolder.h @@ -23,10 +23,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/FMF.h" +#include "llvm/IR/IRBuilderFolder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/IRBuilderFolder.h" namespace llvm { @@ -43,144 +44,72 @@ public: // Return an existing value or a constant if the operation can be simplified. // Otherwise return nullptr. //===--------------------------------------------------------------------===// - Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - return nullptr; - } - Value *FoldAnd(Value *LHS, Value *RHS) const override { return nullptr; } - - Value *FoldOr(Value *LHS, Value *RHS) const override { return nullptr; } - - Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + Value *FoldBinOp(Instruction::BinaryOps Opc, Value *LHS, + Value *RHS) const override { return nullptr; } - Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, - bool IsInBounds = false) const override { + Value *FoldExactBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool IsExact) const override { return nullptr; } - Value *FoldSelect(Value *C, Value *True, Value *False) const override { + Value *FoldNoWrapBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + bool HasNUW, bool HasNSW) const override { return nullptr; } - //===--------------------------------------------------------------------===// - // Binary Operators - //===--------------------------------------------------------------------===// - - Instruction *CreateFAdd(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateFAdd(LHS, RHS); - } - - Instruction *CreateSub(Constant *LHS, Constant *RHS, - bool HasNUW = false, - bool HasNSW = false) const override { - BinaryOperator *BO = BinaryOperator::CreateSub(LHS, RHS); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; - } - - Instruction *CreateFSub(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateFSub(LHS, RHS); - } - - Instruction *CreateMul(Constant *LHS, Constant *RHS, - bool HasNUW = false, - bool HasNSW = false) const override { - BinaryOperator *BO = BinaryOperator::CreateMul(LHS, RHS); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; - } - - Instruction *CreateFMul(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateFMul(LHS, RHS); - } - - Instruction *CreateUDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - if (!isExact) - return BinaryOperator::CreateUDiv(LHS, RHS); - return BinaryOperator::CreateExactUDiv(LHS, RHS); - } - - Instruction *CreateSDiv(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - if (!isExact) - return BinaryOperator::CreateSDiv(LHS, RHS); - return BinaryOperator::CreateExactSDiv(LHS, RHS); - } - - Instruction *CreateFDiv(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateFDiv(LHS, RHS); + Value *FoldBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, + FastMathFlags FMF) const override { + return nullptr; } - Instruction *CreateURem(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateURem(LHS, RHS); + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + return nullptr; } - Instruction *CreateSRem(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateSRem(LHS, RHS); + Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const override { + return nullptr; } - Instruction *CreateFRem(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateFRem(LHS, RHS); + Value *FoldSelect(Value *C, Value *True, Value *False) const override { + return nullptr; } - Instruction *CreateShl(Constant *LHS, Constant *RHS, bool HasNUW = false, - bool HasNSW = false) const override { - BinaryOperator *BO = BinaryOperator::CreateShl(LHS, RHS); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; + Value *FoldExtractValue(Value *Agg, + ArrayRef<unsigned> IdxList) const override { + return nullptr; } - Instruction *CreateLShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - if (!isExact) - return BinaryOperator::CreateLShr(LHS, RHS); - return BinaryOperator::CreateExactLShr(LHS, RHS); + Value *FoldInsertValue(Value *Agg, Value *Val, + ArrayRef<unsigned> IdxList) const override { + return nullptr; } - Instruction *CreateAShr(Constant *LHS, Constant *RHS, - bool isExact = false) const override { - if (!isExact) - return BinaryOperator::CreateAShr(LHS, RHS); - return BinaryOperator::CreateExactAShr(LHS, RHS); + Value *FoldExtractElement(Value *Vec, Value *Idx) const override { + return nullptr; } - Instruction *CreateXor(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateXor(LHS, RHS); + Value *FoldInsertElement(Value *Vec, Value *NewElt, + Value *Idx) const override { + return nullptr; } - Instruction *CreateBinOp(Instruction::BinaryOps Opc, - Constant *LHS, Constant *RHS) const override { - return BinaryOperator::Create(Opc, LHS, RHS); + Value *FoldShuffleVector(Value *V1, Value *V2, + ArrayRef<int> Mask) const override { + return nullptr; } //===--------------------------------------------------------------------===// // Unary Operators //===--------------------------------------------------------------------===// - Instruction *CreateNeg(Constant *C, - bool HasNUW = false, - bool HasNSW = false) const override { - BinaryOperator *BO = BinaryOperator::CreateNeg(C); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; - } - Instruction *CreateFNeg(Constant *C) const override { return UnaryOperator::CreateFNeg(C); } - Instruction *CreateNot(Constant *C) const override { - return BinaryOperator::CreateNot(C); - } - Instruction *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const override { return UnaryOperator::Create(Opc, C); @@ -245,35 +174,6 @@ public: Constant *LHS, Constant *RHS) const override { return new FCmpInst(P, LHS, RHS); } - - //===--------------------------------------------------------------------===// - // Other Instructions - //===--------------------------------------------------------------------===// - - Instruction *CreateExtractElement(Constant *Vec, - Constant *Idx) const override { - return ExtractElementInst::Create(Vec, Idx); - } - - Instruction *CreateInsertElement(Constant *Vec, Constant *NewElt, - Constant *Idx) const override { - return InsertElementInst::Create(Vec, NewElt, Idx); - } - - Instruction *CreateShuffleVector(Constant *V1, Constant *V2, - ArrayRef<int> Mask) const override { - return new ShuffleVectorInst(V1, V2, Mask); - } - - Instruction *CreateExtractValue(Constant *Agg, - ArrayRef<unsigned> IdxList) const override { - return ExtractValueInst::Create(Agg, IdxList); - } - - Instruction *CreateInsertValue(Constant *Agg, Constant *Val, - ArrayRef<unsigned> IdxList) const override { - return InsertValueInst::Create(Agg, Val, IdxList); - } }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h index 7d232bba0864..1a234e273eff 100644 --- a/llvm/include/llvm/IR/Operator.h +++ b/llvm/include/llvm/IR/Operator.h @@ -18,6 +18,7 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/FMF.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -161,105 +162,6 @@ public: } }; -/// Convenience struct for specifying and reasoning about fast-math flags. -class FastMathFlags { -private: - friend class FPMathOperator; - - unsigned Flags = 0; - - FastMathFlags(unsigned F) { - // If all 7 bits are set, turn this into -1. If the number of bits grows, - // this must be updated. This is intended to provide some forward binary - // compatibility insurance for the meaning of 'fast' in case bits are added. - if (F == 0x7F) Flags = ~0U; - else Flags = F; - } - -public: - // This is how the bits are used in Value::SubclassOptionalData so they - // should fit there too. - // WARNING: We're out of space. SubclassOptionalData only has 7 bits. New - // functionality will require a change in how this information is stored. - enum { - AllowReassoc = (1 << 0), - NoNaNs = (1 << 1), - NoInfs = (1 << 2), - NoSignedZeros = (1 << 3), - AllowReciprocal = (1 << 4), - AllowContract = (1 << 5), - ApproxFunc = (1 << 6) - }; - - FastMathFlags() = default; - - static FastMathFlags getFast() { - FastMathFlags FMF; - FMF.setFast(); - return FMF; - } - - bool any() const { return Flags != 0; } - bool none() const { return Flags == 0; } - bool all() const { return Flags == ~0U; } - - void clear() { Flags = 0; } - void set() { Flags = ~0U; } - - /// Flag queries - bool allowReassoc() const { return 0 != (Flags & AllowReassoc); } - bool noNaNs() const { return 0 != (Flags & NoNaNs); } - bool noInfs() const { return 0 != (Flags & NoInfs); } - bool noSignedZeros() const { return 0 != (Flags & NoSignedZeros); } - bool allowReciprocal() const { return 0 != (Flags & AllowReciprocal); } - bool allowContract() const { return 0 != (Flags & AllowContract); } - bool approxFunc() const { return 0 != (Flags & ApproxFunc); } - /// 'Fast' means all bits are set. - bool isFast() const { return all(); } - - /// Flag setters - void setAllowReassoc(bool B = true) { - Flags = (Flags & ~AllowReassoc) | B * AllowReassoc; - } - void setNoNaNs(bool B = true) { - Flags = (Flags & ~NoNaNs) | B * NoNaNs; - } - void setNoInfs(bool B = true) { - Flags = (Flags & ~NoInfs) | B * NoInfs; - } - void setNoSignedZeros(bool B = true) { - Flags = (Flags & ~NoSignedZeros) | B * NoSignedZeros; - } - void setAllowReciprocal(bool B = true) { - Flags = (Flags & ~AllowReciprocal) | B * AllowReciprocal; - } - void setAllowContract(bool B = true) { - Flags = (Flags & ~AllowContract) | B * AllowContract; - } - void setApproxFunc(bool B = true) { - Flags = (Flags & ~ApproxFunc) | B * ApproxFunc; - } - void setFast(bool B = true) { B ? set() : clear(); } - - void operator&=(const FastMathFlags &OtherFlags) { - Flags &= OtherFlags.Flags; - } - void operator|=(const FastMathFlags &OtherFlags) { - Flags |= OtherFlags.Flags; - } - bool operator!=(const FastMathFlags &OtherFlags) const { - return Flags != OtherFlags.Flags; - } - - /// Print fast-math flags to \p O. - void print(raw_ostream &O) const; -}; - -inline raw_ostream &operator<<(raw_ostream &O, FastMathFlags FMF) { - FMF.print(O); - return O; -} - /// Utility class for floating point operations which can have /// information about relaxed accuracy requirements attached to them. class FPMathOperator : public Operator { diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index f9f4f1603861..7f0695b552e1 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -136,7 +136,9 @@ struct undef_match { inline auto m_Undef() { return undef_match(); } /// Match an arbitrary poison constant. -inline class_match<PoisonValue> m_Poison() { return class_match<PoisonValue>(); } +inline class_match<PoisonValue> m_Poison() { + return class_match<PoisonValue>(); +} /// Match an arbitrary Constant and ignore it. inline class_match<Constant> m_Constant() { return class_match<Constant>(); } @@ -222,7 +224,7 @@ struct apint_match { bool AllowUndef; apint_match(const APInt *&Res, bool AllowUndef) - : Res(Res), AllowUndef(AllowUndef) {} + : Res(Res), AllowUndef(AllowUndef) {} template <typename ITy> bool match(ITy *V) { if (auto *CI = dyn_cast<ConstantInt>(V)) { @@ -231,8 +233,8 @@ struct apint_match { } if (V->getType()->isVectorTy()) if (const auto *C = dyn_cast<Constant>(V)) - if (auto *CI = dyn_cast_or_null<ConstantInt>( - C->getSplatValue(AllowUndef))) { + if (auto *CI = + dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndef))) { Res = &CI->getValue(); return true; } @@ -256,8 +258,8 @@ struct apfloat_match { } if (V->getType()->isVectorTy()) if (const auto *C = dyn_cast<Constant>(V)) - if (auto *CI = dyn_cast_or_null<ConstantFP>( - C->getSplatValue(AllowUndef))) { + if (auto *CI = + dyn_cast_or_null<ConstantFP>(C->getSplatValue(AllowUndef))) { Res = &CI->getValueAPF(); return true; } @@ -467,9 +469,7 @@ struct is_negative { inline cst_pred_ty<is_negative> m_Negative() { return cst_pred_ty<is_negative>(); } -inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { - return V; -} +inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { return V; } struct is_nonnegative { bool isValue(const APInt &C) { return C.isNonNegative(); } @@ -479,9 +479,7 @@ struct is_nonnegative { inline cst_pred_ty<is_nonnegative> m_NonNegative() { return cst_pred_ty<is_nonnegative>(); } -inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { - return V; -} +inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; } struct is_strictlypositive { bool isValue(const APInt &C) { return C.isStrictlyPositive(); } @@ -510,9 +508,7 @@ struct is_one { }; /// Match an integer 1 or a vector with all elements equal to 1. /// For vectors, this includes constants with undefined elements. -inline cst_pred_ty<is_one> m_One() { - return cst_pred_ty<is_one>(); -} +inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); } struct is_zero_int { bool isValue(const APInt &C) { return C.isZero(); } @@ -532,21 +528,15 @@ struct is_zero { }; /// Match any null constant or a vector with all elements equal to 0. /// For vectors, this includes constants with undefined elements. -inline is_zero m_Zero() { - return is_zero(); -} +inline is_zero m_Zero() { return is_zero(); } struct is_power2 { bool isValue(const APInt &C) { return C.isPowerOf2(); } }; /// Match an integer or vector power-of-2. /// For vectors, this includes constants with undefined elements. -inline cst_pred_ty<is_power2> m_Power2() { - return cst_pred_ty<is_power2>(); -} -inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { - return V; -} +inline cst_pred_ty<is_power2> m_Power2() { return cst_pred_ty<is_power2>(); } +inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; } struct is_negated_power2 { bool isValue(const APInt &C) { return C.isNegatedPowerOf2(); } @@ -589,9 +579,7 @@ struct is_lowbit_mask { inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { return cst_pred_ty<is_lowbit_mask>(); } -inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { - return V; -} +inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { return V; } struct icmp_pred_with_threshold { ICmpInst::Predicate Pred; @@ -613,9 +601,7 @@ struct is_nan { }; /// Match an arbitrary NaN constant. This includes quiet and signalling nans. /// For vectors, this includes constants with undefined elements. -inline cstfp_pred_ty<is_nan> m_NaN() { - return cstfp_pred_ty<is_nan>(); -} +inline cstfp_pred_ty<is_nan> m_NaN() { return cstfp_pred_ty<is_nan>(); } struct is_nonnan { bool isValue(const APFloat &C) { return !C.isNaN(); } @@ -631,9 +617,7 @@ struct is_inf { }; /// Match a positive or negative infinity FP constant. /// For vectors, this includes constants with undefined elements. -inline cstfp_pred_ty<is_inf> m_Inf() { - return cstfp_pred_ty<is_inf>(); -} +inline cstfp_pred_ty<is_inf> m_Inf() { return cstfp_pred_ty<is_inf>(); } struct is_noninf { bool isValue(const APFloat &C) { return !C.isInfinity(); } @@ -729,7 +713,9 @@ inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } /// Match a binary operator, capturing it if we match. inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } /// Match a with overflow intrinsic, capturing it if we match. -inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; } +inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { + return I; +} inline bind_ty<const WithOverflowInst> m_WithOverflowInst(const WithOverflowInst *&I) { return I; @@ -842,8 +828,7 @@ struct bind_const_intval_ty { /// Match a specified integer value or vector of all elements of that /// value. -template <bool AllowUndefs> -struct specific_intval { +template <bool AllowUndefs> struct specific_intval { APInt Val; specific_intval(APInt V) : Val(std::move(V)) {} @@ -1014,7 +999,8 @@ template <typename Op_t> struct FNeg_match { FNeg_match(const Op_t &Op) : X(Op) {} template <typename OpTy> bool match(OpTy *V) { auto *FPMO = dyn_cast<FPMathOperator>(V); - if (!FPMO) return false; + if (!FPMO) + return false; if (FPMO->getOpcode() == Instruction::FNeg) return X.match(FPMO->getOperand(0)); @@ -1038,9 +1024,7 @@ template <typename Op_t> struct FNeg_match { }; /// Match 'fneg X' as 'fsub -0.0, X'. -template <typename OpTy> -inline FNeg_match<OpTy> -m_FNeg(const OpTy &X) { +template <typename OpTy> inline FNeg_match<OpTy> m_FNeg(const OpTy &X) { return FNeg_match<OpTy>(X); } @@ -1165,32 +1149,32 @@ inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap> m_NSWAdd(const LHS &L, const RHS &R) { return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, - OverflowingBinaryOperator::NoSignedWrap>( - L, R); + OverflowingBinaryOperator::NoSignedWrap>(L, + R); } template <typename LHS, typename RHS> inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap> m_NSWSub(const LHS &L, const RHS &R) { return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, - OverflowingBinaryOperator::NoSignedWrap>( - L, R); + OverflowingBinaryOperator::NoSignedWrap>(L, + R); } template <typename LHS, typename RHS> inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoSignedWrap> m_NSWMul(const LHS &L, const RHS &R) { return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, - OverflowingBinaryOperator::NoSignedWrap>( - L, R); + OverflowingBinaryOperator::NoSignedWrap>(L, + R); } template <typename LHS, typename RHS> inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoSignedWrap> m_NSWShl(const LHS &L, const RHS &R) { return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, - OverflowingBinaryOperator::NoSignedWrap>( - L, R); + OverflowingBinaryOperator::NoSignedWrap>(L, + R); } template <typename LHS, typename RHS> @@ -1384,7 +1368,7 @@ struct CmpClass_match { Predicate = I->getPredicate(); return true; } else if (Commutable && L.match(I->getOperand(1)) && - R.match(I->getOperand(0))) { + R.match(I->getOperand(0))) { Predicate = I->getSwappedPredicate(); return true; } @@ -2080,15 +2064,13 @@ template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { }; template <typename T0, typename T1, typename T2> struct m_Intrinsic_Ty<T0, T1, T2> { - using Ty = - match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, - Argument_match<T2>>; + using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, + Argument_match<T2>>; }; template <typename T0, typename T1, typename T2, typename T3> struct m_Intrinsic_Ty<T0, T1, T2, T3> { - using Ty = - match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, - Argument_match<T3>>; + using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, + Argument_match<T3>>; }; template <typename T0, typename T1, typename T2, typename T3, typename T4> @@ -2097,7 +2079,8 @@ struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { Argument_match<T4>>; }; -template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5> +template <typename T0, typename T1, typename T2, typename T3, typename T4, + typename T5> struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, Argument_match<T5>>; @@ -2117,6 +2100,14 @@ m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); } +/// Matches MaskedGather Intrinsic. +template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> +inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty +m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, + const Opnd3 &Op3) { + return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2, Op3); +} + template <Intrinsic::ID IntrID, typename T0> inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); @@ -2204,6 +2195,11 @@ m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); } +template <typename Opnd0> +inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) { + return m_Intrinsic<Intrinsic::sqrt>(Op0); +} + //===----------------------------------------------------------------------===// // Matchers for two-operands operators with the operators in either order // @@ -2532,8 +2528,8 @@ struct LogicalOp_match { /// Matches L && R either in the form of L & R or L ? R : false. /// Note that the latter form is poison-blocking. template <typename LHS, typename RHS> -inline LogicalOp_match<LHS, RHS, Instruction::And> -m_LogicalAnd(const LHS &L, const RHS &R) { +inline LogicalOp_match<LHS, RHS, Instruction::And> m_LogicalAnd(const LHS &L, + const RHS &R) { return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); } @@ -2550,8 +2546,8 @@ m_c_LogicalAnd(const LHS &L, const RHS &R) { /// Matches L || R either in the form of L | R or L ? true : R. /// Note that the latter form is poison-blocking. template <typename LHS, typename RHS> -inline LogicalOp_match<LHS, RHS, Instruction::Or> -m_LogicalOr(const LHS &L, const RHS &R) { +inline LogicalOp_match<LHS, RHS, Instruction::Or> m_LogicalOr(const LHS &L, + const RHS &R) { return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); } diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def index 62d67308114f..39c11771ff41 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.def +++ b/llvm/include/llvm/IR/RuntimeLibcalls.def @@ -47,6 +47,8 @@ HANDLE_LIBCALL(MUL_I16, "__mulhi3") HANDLE_LIBCALL(MUL_I32, "__mulsi3") HANDLE_LIBCALL(MUL_I64, "__muldi3") HANDLE_LIBCALL(MUL_I128, "__multi3") +HANDLE_LIBCALL(MUL_IEXT, nullptr) + HANDLE_LIBCALL(MULO_I32, "__mulosi4") HANDLE_LIBCALL(MULO_I64, "__mulodi4") HANDLE_LIBCALL(MULO_I128, "__muloti4") @@ -55,31 +57,43 @@ HANDLE_LIBCALL(SDIV_I16, "__divhi3") HANDLE_LIBCALL(SDIV_I32, "__divsi3") HANDLE_LIBCALL(SDIV_I64, "__divdi3") HANDLE_LIBCALL(SDIV_I128, "__divti3") +HANDLE_LIBCALL(SDIV_IEXT, "__divei4") + HANDLE_LIBCALL(UDIV_I8, "__udivqi3") HANDLE_LIBCALL(UDIV_I16, "__udivhi3") HANDLE_LIBCALL(UDIV_I32, "__udivsi3") HANDLE_LIBCALL(UDIV_I64, "__udivdi3") HANDLE_LIBCALL(UDIV_I128, "__udivti3") +HANDLE_LIBCALL(UDIV_IEXT, "__udivei4") + HANDLE_LIBCALL(SREM_I8, "__modqi3") HANDLE_LIBCALL(SREM_I16, "__modhi3") HANDLE_LIBCALL(SREM_I32, "__modsi3") HANDLE_LIBCALL(SREM_I64, "__moddi3") HANDLE_LIBCALL(SREM_I128, "__modti3") +HANDLE_LIBCALL(SREM_IEXT, "__modei4") + HANDLE_LIBCALL(UREM_I8, "__umodqi3") HANDLE_LIBCALL(UREM_I16, "__umodhi3") HANDLE_LIBCALL(UREM_I32, "__umodsi3") HANDLE_LIBCALL(UREM_I64, "__umoddi3") HANDLE_LIBCALL(UREM_I128, "__umodti3") +HANDLE_LIBCALL(UREM_IEXT, "__umodei4") + HANDLE_LIBCALL(SDIVREM_I8, nullptr) HANDLE_LIBCALL(SDIVREM_I16, nullptr) HANDLE_LIBCALL(SDIVREM_I32, nullptr) HANDLE_LIBCALL(SDIVREM_I64, nullptr) HANDLE_LIBCALL(SDIVREM_I128, nullptr) +HANDLE_LIBCALL(SDIVREM_IEXT, nullptr) + HANDLE_LIBCALL(UDIVREM_I8, nullptr) HANDLE_LIBCALL(UDIVREM_I16, nullptr) HANDLE_LIBCALL(UDIVREM_I32, nullptr) HANDLE_LIBCALL(UDIVREM_I64, nullptr) HANDLE_LIBCALL(UDIVREM_I128, nullptr) +HANDLE_LIBCALL(UDIVREM_IEXT, nullptr) + HANDLE_LIBCALL(NEG_I32, "__negsi2") HANDLE_LIBCALL(NEG_I64, "__negdi2") HANDLE_LIBCALL(CTLZ_I32, "__clzsi2") @@ -296,6 +310,8 @@ HANDLE_LIBCALL(FPROUND_F64_F16, "__truncdfhf2") HANDLE_LIBCALL(FPROUND_F80_F16, "__truncxfhf2") HANDLE_LIBCALL(FPROUND_F128_F16, "__trunctfhf2") HANDLE_LIBCALL(FPROUND_PPCF128_F16, "__trunctfhf2") +HANDLE_LIBCALL(FPROUND_F32_BF16, "__truncsfbf2") +HANDLE_LIBCALL(FPROUND_F64_BF16, "__truncdfbf2") HANDLE_LIBCALL(FPROUND_F64_F32, "__truncdfsf2") HANDLE_LIBCALL(FPROUND_F80_F32, "__truncxfsf2") HANDLE_LIBCALL(FPROUND_F128_F32, "__trunctfsf2") diff --git a/llvm/include/llvm/IR/Statepoint.h b/llvm/include/llvm/IR/Statepoint.h index da9c732ad818..ba8ffbbaf397 100644 --- a/llvm/include/llvm/IR/Statepoint.h +++ b/llvm/include/llvm/IR/Statepoint.h @@ -121,9 +121,8 @@ public: /// Return the type of the value returned by the call underlying the /// statepoint. Type *getActualReturnType() const { - auto *CalleeTy = - getActualCalledOperand()->getType()->getPointerElementType(); - return cast<FunctionType>(CalleeTy)->getReturnType(); + auto *FT = cast<FunctionType>(getParamElementType(CalledFunctionPos)); + return FT->getReturnType(); } diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h index e4e8a5529c87..51263c6b8fcc 100644 --- a/llvm/include/llvm/IR/Type.h +++ b/llvm/include/llvm/IR/Type.h @@ -68,13 +68,14 @@ public: TokenTyID, ///< Tokens // Derived types... see DerivedTypes.h file. - IntegerTyID, ///< Arbitrary bit width integers - FunctionTyID, ///< Functions - PointerTyID, ///< Pointers - StructTyID, ///< Structures - ArrayTyID, ///< Arrays - FixedVectorTyID, ///< Fixed width SIMD vector type - ScalableVectorTyID ///< Scalable SIMD vector type + IntegerTyID, ///< Arbitrary bit width integers + FunctionTyID, ///< Functions + PointerTyID, ///< Pointers + StructTyID, ///< Structures + ArrayTyID, ///< Arrays + FixedVectorTyID, ///< Fixed width SIMD vector type + ScalableVectorTyID, ///< Scalable SIMD vector type + DXILPointerTyID, ///< DXIL typed pointer used by DirectX target }; private: @@ -368,6 +369,9 @@ public: /// This method is deprecated without replacement. Pointer element types are /// not available with opaque pointers. + [[deprecated("Deprecated without replacement, see " + "https://llvm.org/docs/OpaquePointers.html for context and " + "migration instructions")]] Type *getPointerElementType() const { return getNonOpaquePointerElementType(); } diff --git a/llvm/include/llvm/IR/User.h b/llvm/include/llvm/IR/User.h index 221bb5b2cb1c..a9cf60151e5d 100644 --- a/llvm/include/llvm/IR/User.h +++ b/llvm/include/llvm/IR/User.h @@ -304,8 +304,8 @@ public: /// Replace uses of one Value with another. /// /// Replaces all references to the "From" definition with references to the - /// "To" definition. - void replaceUsesOfWith(Value *From, Value *To); + /// "To" definition. Returns whether any uses were replaced. + bool replaceUsesOfWith(Value *From, Value *To); // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def index 1abcbb874a8d..1d639e8aeb01 100644 --- a/llvm/include/llvm/IR/VPIntrinsics.def +++ b/llvm/include/llvm/IR/VPIntrinsics.def @@ -54,6 +54,12 @@ #define END_REGISTER_VP_SDNODE(VPSD) #endif +// Helper macro to set up the mapping from VP intrinsic to ISD opcode. +// Note: More than one VP intrinsic may map to one ISD opcode. +#ifndef HELPER_MAP_VPID_TO_VPSD +#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) +#endif + // Helper macros for the common "1:1 - Intrinsic : SDNode" case. // // There is one VP intrinsic that maps directly to one SDNode that goes by the @@ -70,7 +76,8 @@ // the SDNode is used. #define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \ BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \ - BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) + BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \ + HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) #define END_REGISTER_VP(VPID, VPSD) \ END_REGISTER_VP_INTRINSIC(VPID) \ @@ -121,6 +128,18 @@ #define VP_PROPERTY_BINARYOP #endif +// A property to infer VP type casts automatically. +#ifndef VP_PROPERTY_CASTOP +#define VP_PROPERTY_CASTOP +#endif + +// This VP Intrinsic is a comparison operation +// The condition code arg is at CCPOS and accepts floating-point condition +// codes if ISFP is set, else it accepts integer condition codes. +#ifndef VP_PROPERTY_CMP +#define VP_PROPERTY_CMP(CCPOS, ISFP) +#endif + /// } Property Macros ///// Integer Arithmetic { @@ -211,22 +230,130 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem) #undef HELPER_REGISTER_BINARY_FP_VP +// llvm.vp.fneg(x,mask,vlen) +BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1) +VP_PROPERTY_FUNCTIONAL_OPC(FNeg) +END_REGISTER_VP(vp_fneg, VP_FNEG) + +// llvm.vp.fma(x,y,z,mask,vlen) +BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1) +VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fma) +END_REGISTER_VP(vp_fma, VP_FMA) + ///// } Floating-Point Arithmetic +///// Type Casts { +// Specialized helper macro for type conversions. +// <operation>(%x, %mask, %evl). +#ifdef HELPER_REGISTER_FP_CAST_VP +#error \ + "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!" +#endif +#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, HASROUND) \ + BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ + VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ + VP_PROPERTY_CONSTRAINEDFP(HASROUND, 1, experimental_constrained_##OPSUFFIX) \ + VP_PROPERTY_CASTOP \ + END_REGISTER_VP(vp_##OPSUFFIX, VPSD) + +// llvm.vp.fptoui(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FPTOUI, FPToUI, 0) + +// llvm.vp.fptosi(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FPTOSI, FPToSI, 0) + +// llvm.vp.uitofp(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UITOFP, UIToFP, 1) + +// llvm.vp.sitofp(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SITOFP, SIToFP, 1) + +// llvm.vp.fptrunc(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, 1) + +// llvm.vp.fpext(x,mask,vlen) +HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, 0) + +#undef HELPER_REGISTER_FP_CAST_VP + +// Specialized helper macro for integer type conversions. +// <operation>(%x, %mask, %evl). +#ifdef HELPER_REGISTER_INT_CAST_VP +#error \ + "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!" +#endif +#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC) \ + BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ + VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ + VP_PROPERTY_CASTOP \ + END_REGISTER_VP(vp_##OPSUFFIX, VPSD) + +// llvm.vp.trunc(x,mask,vlen) +HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc) + +// llvm.vp.zext(x,mask,vlen) +HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt) + +// llvm.vp.sext(x,mask,vlen) +HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt) + +// llvm.vp.ptrtoint(x,mask,vlen) +HELPER_REGISTER_INT_CAST_VP(ptrtoint, VP_PTRTOINT, PtrToInt) + +// llvm.vp.inttoptr(x,mask,vlen) +HELPER_REGISTER_INT_CAST_VP(inttoptr, VP_INTTOPTR, IntToPtr) + +#undef HELPER_REGISTER_INT_CAST_VP + +///// } Type Casts + +///// Comparisons { + +// VP_SETCC (ISel only) +BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4) +END_REGISTER_VP_SDNODE(VP_SETCC) + +// llvm.vp.fcmp(x,y,cc,mask,vlen) +BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4) +HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC) +VP_PROPERTY_FUNCTIONAL_OPC(FCmp) +VP_PROPERTY_CMP(2, true) +VP_PROPERTY_CONSTRAINEDFP(0, 1, experimental_constrained_fcmp) +END_REGISTER_VP_INTRINSIC(vp_fcmp) + +// llvm.vp.icmp(x,y,cc,mask,vlen) +BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4) +HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC) +VP_PROPERTY_FUNCTIONAL_OPC(ICmp) +VP_PROPERTY_CMP(2, false) +END_REGISTER_VP_INTRINSIC(vp_icmp) + +///// } Comparisons + ///// Memory Operations { // llvm.vp.store(val,ptr,mask,vlen) BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3) // chain = VP_STORE chain,val,base,offset,mask,evl BEGIN_REGISTER_VP_SDNODE(VP_STORE, 0, vp_store, 4, 5) +HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE) VP_PROPERTY_FUNCTIONAL_OPC(Store) VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store) VP_PROPERTY_MEMOP(1, 0) END_REGISTER_VP(vp_store, VP_STORE) +// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen) +BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4) +// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl +BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 0, experimental_vp_strided_store, 5, 6) +HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) +VP_PROPERTY_MEMOP(1, 0) +END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) + // llvm.vp.scatter(ptr,val,mask,vlen) BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3) // chain = VP_SCATTER chain,val,base,indices,scale,mask,evl BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, -1, vp_scatter, 5, 6) +HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER) VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter) VP_PROPERTY_MEMOP(1, 0) END_REGISTER_VP(vp_scatter, VP_SCATTER) @@ -235,15 +362,25 @@ END_REGISTER_VP(vp_scatter, VP_SCATTER) BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2) // val,chain = VP_LOAD chain,base,offset,mask,evl BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4) +HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD) VP_PROPERTY_FUNCTIONAL_OPC(Load) VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load) VP_PROPERTY_MEMOP(0, None) END_REGISTER_VP(vp_load, VP_LOAD) +// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen) +BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3) +// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl +BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5) +HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) +VP_PROPERTY_MEMOP(0, None) +END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) + // llvm.vp.gather(ptr,mask,vlen) BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2) // val,chain = VP_GATHER chain,base,indices,scale,mask,evl BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5) +HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER) VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather) VP_PROPERTY_MEMOP(0, None) END_REGISTER_VP(vp_gather, VP_GATHER) @@ -313,6 +450,8 @@ HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, // sequential and reassociative. These manifest as the presence of 'reassoc' // fast-math flags in the IR and as two distinct ISD opcodes in the // SelectionDAG. +// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which +// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set. #ifdef HELPER_REGISTER_REDUCTION_SEQ_VP #error \ "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!" @@ -323,6 +462,7 @@ HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, VP_PROPERTY_REDUCTION(0, 1) \ END_REGISTER_VP_SDNODE(VPSD) \ BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, -1, VPID, 2, 3) \ + HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \ VP_PROPERTY_REDUCTION(0, 1) \ END_REGISTER_VP_SDNODE(SEQ_VPSD) \ VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ @@ -344,13 +484,18 @@ HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL, ///// Shuffles { -// llvm.vp.select(mask,on_true,on_false,vlen) -BEGIN_REGISTER_VP(vp_select, 0, 3, VP_SELECT, -1) +// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported +// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other +// VP intrinsics, these two have a defined result on lanes where the mask is +// false. +// +// llvm.vp.select(cond,on_true,on_false,vlen) +BEGIN_REGISTER_VP(vp_select, None, 3, VP_SELECT, -1) VP_PROPERTY_FUNCTIONAL_OPC(Select) END_REGISTER_VP(vp_select, VP_SELECT) -// llvm.vp.merge(mask,on_true,on_false,pivot) -BEGIN_REGISTER_VP(vp_merge, 0, 3, VP_MERGE, -1) +// llvm.vp.merge(cond,on_true,on_false,pivot) +BEGIN_REGISTER_VP(vp_merge, None, 3, VP_MERGE, -1) END_REGISTER_VP(vp_merge, VP_MERGE) BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) @@ -364,7 +509,10 @@ END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) #undef END_REGISTER_VP #undef END_REGISTER_VP_INTRINSIC #undef END_REGISTER_VP_SDNODE +#undef HELPER_MAP_VPID_TO_VPSD #undef VP_PROPERTY_BINARYOP +#undef VP_PROPERTY_CASTOP +#undef VP_PROPERTY_CMP #undef VP_PROPERTY_CONSTRAINEDFP #undef VP_PROPERTY_FUNCTIONAL_INTRINSIC #undef VP_PROPERTY_FUNCTIONAL_OPC diff --git a/llvm/include/llvm/IR/ValueMap.h b/llvm/include/llvm/IR/ValueMap.h index 67f275cc06d9..a4b6091cf115 100644 --- a/llvm/include/llvm/IR/ValueMap.h +++ b/llvm/include/llvm/IR/ValueMap.h @@ -104,8 +104,8 @@ public: : Map(NumInitBuckets), Data() {} explicit ValueMap(const ExtraData &Data, unsigned NumInitBuckets = 64) : Map(NumInitBuckets), Data(Data) {} - // ValueMap can't be copied nor moved, beucase the callbacks store pointer - // to it. + // ValueMap can't be copied nor moved, because the callbacks store pointer to + // it. ValueMap(const ValueMap &) = delete; ValueMap(ValueMap &&) = delete; ValueMap &operator=(const ValueMap &) = delete; @@ -141,7 +141,7 @@ public: size_type size() const { return Map.size(); } /// Grow the map so that it has at least Size buckets. Does not shrink - void resize(size_t Size) { Map.resize(Size); } + void reserve(size_t Size) { Map.reserve(Size); } void clear() { Map.clear(); diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h new file mode 100644 index 000000000000..301edaed70fe --- /dev/null +++ b/llvm/include/llvm/IR/VectorBuilder.h @@ -0,0 +1,99 @@ +//===- llvm/VectorBuilder.h - Builder for VP Intrinsics ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the VectorBuilder class, which is used as a convenient way +// to create VP intrinsics as if they were LLVM instructions with a consistent +// and simplified interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_VECTORBUILDER_H +#define LLVM_IR_VECTORBUILDER_H + +#include <llvm/IR/IRBuilder.h> +#include <llvm/IR/InstrTypes.h> +#include <llvm/IR/Instruction.h> +#include <llvm/IR/Value.h> + +namespace llvm { + +class VectorBuilder { +public: + enum class Behavior { + // Abort if the requested VP intrinsic could not be created. + // This is useful for strict consistency. + ReportAndAbort = 0, + + // Return a default-initialized value if the requested VP intrinsic could + // not be created. + // This is useful for a defensive fallback to non-VP code. + SilentlyReturnNone = 1, + }; + +private: + IRBuilderBase &Builder; + Behavior ErrorHandling; + + // Explicit mask parameter. + Value *Mask; + // Explicit vector length parameter. + Value *ExplicitVectorLength; + // Compile-time vector length. + ElementCount StaticVectorLength; + + // Get mask/evl value handles for the current configuration. + Value &requestMask(); + Value &requestEVL(); + + void handleError(const char *ErrorMsg) const; + template <typename RetType> + RetType returnWithError(const char *ErrorMsg) const { + handleError(ErrorMsg); + return RetType(); + } + +public: + VectorBuilder(IRBuilderBase &Builder, + Behavior ErrorHandling = Behavior::ReportAndAbort) + : Builder(Builder), ErrorHandling(ErrorHandling), Mask(nullptr), + ExplicitVectorLength(nullptr), + StaticVectorLength(ElementCount::getFixed(0)) {} + + Module &getModule() const; + LLVMContext &getContext() const { return Builder.getContext(); } + + // All-true mask for the currently configured explicit vector length. + Value *getAllTrueMask(); + + VectorBuilder &setMask(Value *NewMask) { + Mask = NewMask; + return *this; + } + VectorBuilder &setEVL(Value *NewExplicitVectorLength) { + ExplicitVectorLength = NewExplicitVectorLength; + return *this; + } + VectorBuilder &setStaticVL(unsigned NewFixedVL) { + StaticVectorLength = ElementCount::getFixed(NewFixedVL); + return *this; + } + // TODO: setStaticVL(ElementCount) for scalable types. + + // Emit a VP intrinsic call that mimics a regular instruction. + // This operation behaves according to the VectorBuilderBehavior. + // \p Opcode The functional instruction opcode of the emitted intrinsic. + // \p ReturnTy The return type of the operation. + // \p VecOpArray The operand list. + Value *createVectorInstruction(unsigned Opcode, Type *ReturnTy, + ArrayRef<Value *> VecOpArray, + const Twine &Name = Twine()); +}; + +} // namespace llvm + +#endif // LLVM_IR_VECTORBUILDER_H diff --git a/llvm/include/llvm/IRReader/IRReader.h b/llvm/include/llvm/IRReader/IRReader.h index a14e46e2edc8..3f2a01fdc54a 100644 --- a/llvm/include/llvm/IRReader/IRReader.h +++ b/llvm/include/llvm/IRReader/IRReader.h @@ -14,7 +14,9 @@ #ifndef LLVM_IRREADER_IRREADER_H #define LLVM_IRREADER_IRREADER_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLForwardCompat.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" #include <memory> diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 489ef045796f..77f2c6330788 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -48,9 +48,6 @@ void initializeInstrumentation(PassRegistry&); /// Initialize all passes linked into the Analysis library. void initializeAnalysis(PassRegistry&); -/// Initialize all passes linked into the Coroutines library. -void initializeCoroutines(PassRegistry&); - /// Initialize all passes linked into the CodeGen library. void initializeCodeGen(PassRegistry&); @@ -65,9 +62,6 @@ void initializeAAResultsWrapperPassPass(PassRegistry&); void initializeADCELegacyPassPass(PassRegistry&); void initializeAddDiscriminatorsLegacyPassPass(PassRegistry&); void initializeAddFSDiscriminatorsPass(PassRegistry &); -void initializeModuleAddressSanitizerLegacyPassPass(PassRegistry &); -void initializeASanGlobalsMetadataWrapperPassPass(PassRegistry &); -void initializeAddressSanitizerLegacyPassPass(PassRegistry &); void initializeAggressiveInstCombinerLegacyPassPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); @@ -77,11 +71,11 @@ void initializeAssumeBuilderPassLegacyPassPass(PassRegistry &); void initializeAnnotation2MetadataLegacyPass(PassRegistry &); void initializeAnnotationRemarksLegacyPass(PassRegistry &); void initializeOpenMPOptCGSCCLegacyPassPass(PassRegistry &); -void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); void initializeAttributorLegacyPassPass(PassRegistry&); void initializeAttributorCGSCCLegacyPassPass(PassRegistry &); +void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &); void initializeBasicBlockSectionsPass(PassRegistry &); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); @@ -103,6 +97,7 @@ void initializeCFGSimplifyPassPass(PassRegistry&); void initializeCFGuardPass(PassRegistry&); void initializeCFGuardLongjmpPass(PassRegistry&); void initializeCFGViewerLegacyPassPass(PassRegistry&); +void initializeCFIFixupPass(PassRegistry&); void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); @@ -137,10 +132,10 @@ void initializeDependenceAnalysisPass(PassRegistry&); void initializeDependenceAnalysisWrapperPassPass(PassRegistry&); void initializeDetectDeadLanesPass(PassRegistry&); void initializeDivRemPairsLegacyPassPass(PassRegistry&); -void initializeDomOnlyPrinterPass(PassRegistry&); -void initializeDomOnlyViewerPass(PassRegistry&); -void initializeDomPrinterPass(PassRegistry&); -void initializeDomViewerPass(PassRegistry&); +void initializeDomOnlyPrinterWrapperPassPass(PassRegistry &); +void initializeDomOnlyViewerWrapperPassPass(PassRegistry &); +void initializeDomPrinterWrapperPassPass(PassRegistry &); +void initializeDomViewerWrapperPassPass(PassRegistry &); void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &); @@ -174,7 +169,6 @@ void initializeFunctionImportLegacyPassPass(PassRegistry&); void initializeFunctionSpecializationLegacyPassPass(PassRegistry &); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); -void initializeGCOVProfilerLegacyPassPass(PassRegistry&); void initializeGVNHoistLegacyPassPass(PassRegistry&); void initializeGVNLegacyPassPass(PassRegistry&); void initializeGVNSinkLegacyPassPass(PassRegistry&); @@ -188,7 +182,6 @@ void initializeHardwareLoopsPass(PassRegistry&); void initializeMIRProfileLoaderPassPass(PassRegistry &); void initializeMemProfilerLegacyPassPass(PassRegistry &); void initializeHotColdSplittingLegacyPassPass(PassRegistry&); -void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &); void initializeIPSCCPLegacyPassPass(PassRegistry&); void initializeIRCELegacyPassPass(PassRegistry&); void initializeIROutlinerLegacyPassPass(PassRegistry&); @@ -215,6 +208,7 @@ void initializeInterleavedAccessPass(PassRegistry&); void initializeInterleavedLoadCombinePass(PassRegistry &); void initializeInternalizeLegacyPassPass(PassRegistry&); void initializeIntervalPartitionPass(PassRegistry&); +void initializeJMCInstrumenterPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); @@ -273,6 +267,7 @@ void initializeLowerAtomicLegacyPassPass(PassRegistry&); void initializeLowerConstantIntrinsicsPass(PassRegistry&); void initializeLowerEmuTLSPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); +void initializeLowerGlobalDtorsLegacyPassPass(PassRegistry &); void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&); void initializeLowerWidenableConditionLegacyPassPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); @@ -316,7 +311,6 @@ void initializeMemDerefPrinterPass(PassRegistry&); void initializeMemoryDependenceWrapperPassPass(PassRegistry&); void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); -void initializeMemorySanitizerLegacyPassPass(PassRegistry&); void initializeMergeFunctionsLegacyPassPass(PassRegistry&); void initializeMergeICmpsLegacyPassPass(PassRegistry &); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); @@ -339,11 +333,6 @@ void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry&); void initializeOptimizePHIsPass(PassRegistry&); void initializePAEvalPass(PassRegistry&); void initializePEIPass(PassRegistry&); -void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&); -void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&); -void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&); -void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&); -void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerLegacyPassPass(PassRegistry&); void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry&); @@ -353,10 +342,10 @@ void initializePhiValuesWrapperPassPass(PassRegistry&); void initializePhysicalRegisterUsageInfoPass(PassRegistry&); void initializePlaceBackedgeSafepointsImplPass(PassRegistry&); void initializePlaceSafepointsPass(PassRegistry&); -void initializePostDomOnlyPrinterPass(PassRegistry&); -void initializePostDomOnlyViewerPass(PassRegistry&); -void initializePostDomPrinterPass(PassRegistry&); -void initializePostDomViewerPass(PassRegistry&); +void initializePostDomOnlyPrinterWrapperPassPass(PassRegistry &); +void initializePostDomOnlyViewerWrapperPassPass(PassRegistry &); +void initializePostDomPrinterWrapperPassPass(PassRegistry &); +void initializePostDomViewerWrapperPassPass(PassRegistry &); void initializePostDominatorTreeWrapperPassPass(PassRegistry&); void initializePostInlineEntryExitInstrumenterPass(PassRegistry&); void initializePostMachineSchedulerPass(PassRegistry&); @@ -405,6 +394,7 @@ void initializeSROALegacyPassPass(PassRegistry&); void initializeSafeStackLegacyPassPass(PassRegistry&); void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&); +void initializeSelectOptimizePass(PassRegistry &); void initializeModuleSanitizerCoverageLegacyPassPass(PassRegistry &); void initializeScalarEvolutionWrapperPassPass(PassRegistry&); void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &); @@ -443,7 +433,7 @@ void initializeTailDuplicatePass(PassRegistry&); void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&); void initializeTargetPassConfigPass(PassRegistry&); void initializeTargetTransformInfoWrapperPassPass(PassRegistry&); -void initializeThreadSanitizerLegacyPassPass(PassRegistry&); +void initializeTLSVariableHoistLegacyPassPass(PassRegistry &); void initializeTwoAddressInstructionPassPass(PassRegistry&); void initializeTypeBasedAAWrapperPassPass(PassRegistry&); void initializeTypePromotionPass(PassRegistry&); diff --git a/llvm/include/llvm/InterfaceStub/ELFObjHandler.h b/llvm/include/llvm/InterfaceStub/ELFObjHandler.h index 20a02c6d5445..c15838c4ae0a 100644 --- a/llvm/include/llvm/InterfaceStub/ELFObjHandler.h +++ b/llvm/include/llvm/InterfaceStub/ELFObjHandler.h @@ -13,16 +13,15 @@ #ifndef LLVM_INTERFACESTUB_ELFOBJHANDLER_H #define LLVM_INTERFACESTUB_ELFOBJHANDLER_H -#include "llvm/InterfaceStub/IFSStub.h" -#include "llvm/Object/ELFObjectFile.h" -#include "llvm/Object/ELFTypes.h" -#include "llvm/Support/FileSystem.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBufferRef.h" +#include <memory> namespace llvm { -class MemoryBuffer; - namespace ifs { +struct IFSStub; /// Attempt to read a binary ELF file from a MemoryBuffer. Expected<std::unique_ptr<IFSStub>> readELFFile(MemoryBufferRef Buf); diff --git a/llvm/include/llvm/InterfaceStub/IFSHandler.h b/llvm/include/llvm/InterfaceStub/IFSHandler.h index 6ae6a421318e..bfa5692811d7 100644 --- a/llvm/include/llvm/InterfaceStub/IFSHandler.h +++ b/llvm/include/llvm/InterfaceStub/IFSHandler.h @@ -19,6 +19,8 @@ #include "llvm/Support/Error.h" #include "llvm/Support/VersionTuple.h" #include <memory> +#include <string> +#include <vector> namespace llvm { @@ -51,8 +53,8 @@ Error validateIFSTarget(IFSStub &Stub, bool ParseTriple); void stripIFSTarget(IFSStub &Stub, bool StripTriple, bool StripArch, bool StripEndianness, bool StripBitWidth); -/// Strips symbols from IFS symbol table that are undefined. -void stripIFSUndefinedSymbols(IFSStub &Stub); +Error filterIFSSyms(IFSStub &Stub, bool StripUndefined, + const std::vector<std::string> &Exclude = {}); /// Parse llvm triple string into a IFSTarget struct. IFSTarget parseTriple(StringRef TripleStr); diff --git a/llvm/include/llvm/InterfaceStub/IFSStub.h b/llvm/include/llvm/InterfaceStub/IFSStub.h index 8c3cd171b1a2..0f935cd478d5 100644 --- a/llvm/include/llvm/InterfaceStub/IFSStub.h +++ b/llvm/include/llvm/InterfaceStub/IFSStub.h @@ -14,9 +14,8 @@ #ifndef LLVM_INTERFACESTUB_IFSSTUB_H #define LLVM_INTERFACESTUB_IFSSTUB_H -#include "llvm/Support/Error.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/VersionTuple.h" -#include <set> #include <vector> namespace llvm { @@ -54,7 +53,7 @@ struct IFSSymbol { IFSSymbol() = default; explicit IFSSymbol(std::string SymbolName) : Name(std::move(SymbolName)) {} std::string Name; - uint64_t Size; + Optional<uint64_t> Size; IFSSymbolType Type; bool Undefined; bool Weak; diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h index eb793d62907e..54bb82d84d96 100644 --- a/llvm/include/llvm/LTO/Config.h +++ b/llvm/include/llvm/LTO/Config.h @@ -57,8 +57,8 @@ struct Config { unsigned OptLevel = 2; bool DisableVerify = false; - /// Use the new pass manager - bool UseNewPM = LLVM_ENABLE_NEW_PASS_MANAGER; + /// Use the standard optimization pipeline. + bool UseDefaultPipeline = false; /// Flag to indicate that the optimizer should not assume builtins are present /// on the target. @@ -177,6 +177,10 @@ struct Config { /// Add FSAFDO discriminators. bool AddFSDiscriminator = false; + /// Use opaque pointer types. Used to call LLVMContext::setOpaquePointers + /// unless already set by the `-opaque-pointers` commandline option. + bool OpaquePointers = true; + /// If this field is set, LTO will write input file paths and symbol /// resolutions here in llvm-lto2 command line flag format. This can be /// used for testing and for running the LTO pipeline outside of the linker @@ -288,6 +292,8 @@ struct LTOLLVMContext : LLVMContext { enableDebugTypeODRUniquing(); setDiagnosticHandler( std::make_unique<LTOLLVMDiagnosticHandler>(&DiagHandler), true); + if (!hasSetOpaquePointersValue()) + setOpaquePointers(C.OpaquePointers); } DiagnosticHandlerFunction DiagHandler; }; diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 0d085a88a193..ea52226dca16 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -197,7 +197,17 @@ using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>( /// This ThinBackend runs the individual backend jobs in-process. /// The default value means to use one job per hardware core (not hyper-thread). -ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism); +/// OnWrite is callback which receives module identifier and notifies LTO user +/// that index file for the module (and optionally imports file) was created. +/// ShouldEmitIndexFiles being true will write sharded ThinLTO index files +/// to the same path as the input module, with suffix ".thinlto.bc" +/// ShouldEmitImportsFiles is true it also writes a list of imported files to a +/// similar path with ".imports" appended instead. +using IndexWriteCallback = std::function<void(const std::string &)>; +ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, + IndexWriteCallback OnWrite = nullptr, + bool ShouldEmitIndexFiles = false, + bool ShouldEmitImportsFiles = false); /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds @@ -212,7 +222,6 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism); /// the final ThinLTO linking. Can be nullptr. /// OnWrite is callback which receives module identifier and notifies LTO user /// that index file for the module (and optionally imports file) was created. -using IndexWriteCallback = std::function<void(const std::string &)>; ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h index 333f483f29c5..96f82a9276e0 100644 --- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -184,7 +184,7 @@ struct LTOCodeGenerator { void setDisableVerify(bool Value) { Config.DisableVerify = Value; } - void setUseNewPM(bool Value) { Config.UseNewPM = Value; } + void setDebugPassManager(bool Enabled) { Config.DebugPassManager = Enabled; } void setDiagnosticHandler(lto_diagnostic_handler_t, void *); diff --git a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h index be1f3154029c..ab40d88af8c1 100644 --- a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h +++ b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h @@ -225,9 +225,6 @@ public: OptLevel = (NewOptLevel > 3) ? 3 : NewOptLevel; } - /// Enable or disable the new pass manager. - void setUseNewPM(unsigned Enabled) { UseNewPM = Enabled; } - /// Enable or disable debug output for the new pass manager. void setDebugPassManager(unsigned Enabled) { DebugPassManager = Enabled; } @@ -347,10 +344,6 @@ private: /// IR Optimization Level [0-3]. unsigned OptLevel = 3; - /// Flag to indicate whether the new pass manager should be used for IR - /// optimizations. - bool UseNewPM = LLVM_ENABLE_NEW_PASS_MANAGER; - /// Flag to indicate whether debug output should be enabled for the new pass /// manager. bool DebugPassManager = false; diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index c8b9aaeed76a..af5926dcb38b 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -75,7 +75,6 @@ namespace { (void) llvm::createAggressiveInstCombinerPass(); (void) llvm::createBitTrackingDCEPass(); (void)llvm::createOpenMPOptCGSCCLegacyPass(); - (void) llvm::createArgumentPromotionPass(); (void) llvm::createAlignmentFromAssumptionsPass(); (void) llvm::createBasicAAWrapperPass(); (void) llvm::createSCEVAAWrapperPass(); @@ -98,16 +97,10 @@ namespace { (void) llvm::createDeadCodeEliminationPass(); (void) llvm::createDeadStoreEliminationPass(); (void) llvm::createDependenceAnalysisWrapperPass(); - (void) llvm::createDomOnlyPrinterPass(); - (void) llvm::createDomPrinterPass(); - (void) llvm::createDomOnlyViewerPass(); - (void) llvm::createDomViewerPass(); - (void) llvm::createGCOVProfilerPass(); - (void) llvm::createPGOInstrumentationGenLegacyPass(); - (void) llvm::createPGOInstrumentationUseLegacyPass(); - (void) llvm::createPGOInstrumentationGenCreateVarLegacyPass(); - (void) llvm::createPGOIndirectCallPromotionLegacyPass(); - (void) llvm::createPGOMemOPSizeOptLegacyPass(); + (void) llvm::createDomOnlyPrinterWrapperPassPass(); + (void) llvm::createDomPrinterWrapperPassPass(); + (void) llvm::createDomOnlyViewerWrapperPassPass(); + (void) llvm::createDomViewerWrapperPassPass(); (void) llvm::createInstrProfilingLegacyPass(); (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); @@ -123,6 +116,7 @@ namespace { (void) llvm::createInstSimplifyLegacyPass(); (void) llvm::createInstructionCombiningPass(); (void) llvm::createInternalizePass(); + (void) llvm::createJMCInstrumenterPass(); (void) llvm::createLCSSAPass(); (void) llvm::createLegacyDivergenceAnalysisPass(); (void) llvm::createLICMPass(); @@ -138,12 +132,12 @@ namespace { (void) llvm::createLoopRerollPass(); (void) llvm::createLoopUnrollPass(); (void) llvm::createLoopUnrollAndJamPass(); - (void) llvm::createLoopUnswitchPass(); (void) llvm::createLoopVersioningLICMPass(); (void) llvm::createLoopIdiomPass(); (void) llvm::createLoopRotatePass(); (void) llvm::createLowerConstantIntrinsicsPass(); (void) llvm::createLowerExpectIntrinsicPass(); + (void) llvm::createLowerGlobalDtorsLegacyPass(); (void) llvm::createLowerInvokePass(); (void) llvm::createLowerSwitchPass(); (void) llvm::createNaryReassociatePass(); @@ -156,10 +150,10 @@ namespace { (void) llvm::createPromoteMemoryToRegisterPass(); (void) llvm::createDemoteRegisterToMemoryPass(); (void) llvm::createPruneEHPass(); - (void) llvm::createPostDomOnlyPrinterPass(); - (void) llvm::createPostDomPrinterPass(); - (void) llvm::createPostDomOnlyViewerPass(); - (void) llvm::createPostDomViewerPass(); + (void)llvm::createPostDomOnlyPrinterWrapperPassPass(); + (void)llvm::createPostDomPrinterWrapperPassPass(); + (void)llvm::createPostDomOnlyViewerWrapperPassPass(); + (void)llvm::createPostDomViewerWrapperPassPass(); (void) llvm::createReassociatePass(); (void) llvm::createRedundantDbgInstEliminationPass(); (void) llvm::createRegionInfoPass(); @@ -176,6 +170,7 @@ namespace { (void) llvm::createStripDeadDebugInfoPass(); (void) llvm::createStripDeadPrototypesPass(); (void) llvm::createTailCallEliminationPass(); + (void)llvm::createTLSVariableHoistPass(); (void) llvm::createJumpThreadingPass(); (void) llvm::createDFAJumpThreadingPass(); (void) llvm::createUnifyFunctionExitNodesPass(); @@ -236,6 +231,7 @@ namespace { (void) llvm::createUnifyLoopExitsPass(); (void) llvm::createFixIrreduciblePass(); (void)llvm::createFunctionSpecializationPass(); + (void)llvm::createSelectOptimizePass(); (void)new llvm::IntervalPartition(); (void)new llvm::ScalarEvolutionWrapperPass(); diff --git a/llvm/include/llvm/Linker/IRMover.h b/llvm/include/llvm/Linker/IRMover.h index e5df83f01fe3..1e3c5394ffa2 100644 --- a/llvm/include/llvm/Linker/IRMover.h +++ b/llvm/include/llvm/Linker/IRMover.h @@ -11,6 +11,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/FunctionExtras.h" #include <functional> namespace llvm { @@ -62,6 +63,8 @@ public: IRMover(Module &M); typedef std::function<void(GlobalValue &)> ValueAdder; + using LazyCallback = + llvm::unique_function<void(GlobalValue &GV, ValueAdder Add)>; /// Move in the provide values in \p ValuesToLink from \p Src. /// @@ -70,11 +73,11 @@ public: /// not present in ValuesToLink. The GlobalValue and a ValueAdder callback /// are passed as an argument, and the callback is expected to be called /// if the GlobalValue needs to be added to the \p ValuesToLink and linked. + /// Pass nullptr if there's no work to be done in such cases. /// - \p IsPerformingImport is true when this IR link is to perform ThinLTO /// function importing from Src. Error move(std::unique_ptr<Module> Src, ArrayRef<GlobalValue *> ValuesToLink, - std::function<void(GlobalValue &GV, ValueAdder Add)> AddLazyFor, - bool IsPerformingImport); + LazyCallback AddLazyFor, bool IsPerformingImport); Module &getModule() { return Composite; } private: diff --git a/llvm/include/llvm/MC/ConstantPools.h b/llvm/include/llvm/MC/ConstantPools.h index 9fe0cce8d68c..7eac75362eff 100644 --- a/llvm/include/llvm/MC/ConstantPools.h +++ b/llvm/include/llvm/MC/ConstantPools.h @@ -43,7 +43,8 @@ struct ConstantPoolEntry { class ConstantPool { using EntryVecTy = SmallVector<ConstantPoolEntry, 4>; EntryVecTy Entries; - std::map<int64_t, const MCSymbolRefExpr *> CachedEntries; + std::map<int64_t, const MCSymbolRefExpr *> CachedConstantEntries; + DenseMap<const MCSymbol *, const MCSymbolRefExpr *> CachedSymbolEntries; public: // Initialize a new empty constant pool diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index bb57c3453d10..a5e7b3f504f5 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -13,12 +13,17 @@ #include "llvm/ADT/Optional.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCFixup.h" -#include "llvm/MC/MCFragment.h" #include "llvm/Support/Endian.h" #include <cstdint> namespace llvm { +class MCAlignFragment; +class MCDwarfCallFrameFragment; +class MCDwarfLineAddrFragment; +class MCFragment; +class MCRelaxableFragment; +class MCSymbol; class MCAsmLayout; class MCAssembler; class MCCFIInstruction; @@ -31,6 +36,7 @@ class MCSubtargetInfo; class MCValue; class raw_pwrite_stream; class StringRef; +class raw_ostream; /// Generic interface to target specific assembler backends. class MCAsmBackend { diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h index 355f569861d8..ec17131e17e8 100644 --- a/llvm/include/llvm/MC/MCAsmInfo.h +++ b/llvm/include/llvm/MC/MCAsmInfo.h @@ -430,6 +430,10 @@ protected: /// hidden visibility. Defaults to MCSA_Hidden. MCSymbolAttr HiddenVisibilityAttr = MCSA_Hidden; + /// This attribute, if not MCSA_Invalid, is used to declare a symbol as having + /// exported visibility. Defaults to MCSA_Exported. + MCSymbolAttr ExportedVisibilityAttr = MCSA_Exported; + /// This attribute, if not MCSA_Invalid, is used to declare an undefined /// symbol as having hidden visibility. Defaults to MCSA_Hidden. MCSymbolAttr HiddenDeclarationVisibilityAttr = MCSA_Hidden; @@ -466,6 +470,10 @@ protected: /// the .loc/.file directives. Defaults to true. bool UsesDwarfFileAndLocDirectives = true; + /// True if DWARF `.file directory' directive syntax is used by + /// default. + bool EnableDwarfFileDirectoryDefault = true; + /// True if the target needs the DWARF section length in the header (if any) /// of the DWARF section in the assembly file. Defaults to true. bool DwarfSectionSizeRequired = true; @@ -478,6 +486,10 @@ protected: /// For example, foo(plt) instead of foo@plt. Defaults to false. bool UseParensForSymbolVariant = false; + /// True if the target uses parens for symbol names starting with + /// '$' character to distinguish them from absolute names. + bool UseParensForDollarSignNames = true; + /// True if the target supports flags in ".loc" directive, false if only /// location is allowed. bool SupportsExtendedDwarfLocDirective = true; @@ -671,6 +683,7 @@ public: const char *getCode64Directive() const { return Code64Directive; } unsigned getAssemblerDialect() const { return AssemblerDialect; } bool doesAllowAtInName() const { return AllowAtInName; } + void setAllowAtInName(bool V) { AllowAtInName = V; } bool doesAllowQuestionAtStartOfIdentifier() const { return AllowQuestionAtStartOfIdentifier; } @@ -749,6 +762,8 @@ public: MCSymbolAttr getHiddenVisibilityAttr() const { return HiddenVisibilityAttr; } + MCSymbolAttr getExportedVisibilityAttr() const { return ExportedVisibilityAttr; } + MCSymbolAttr getHiddenDeclarationVisibilityAttr() const { return HiddenDeclarationVisibilityAttr; } @@ -788,6 +803,9 @@ public: bool doDwarfFDESymbolsUseAbsDiff() const { return DwarfFDESymbolsUseAbsDiff; } bool useDwarfRegNumForCFI() const { return DwarfRegNumForCFI; } bool useParensForSymbolVariant() const { return UseParensForSymbolVariant; } + bool useParensForDollarSignNames() const { + return UseParensForDollarSignNames; + } bool supportsExtendedDwarfLocDirective() const { return SupportsExtendedDwarfLocDirective; } @@ -800,6 +818,10 @@ public: return DwarfSectionSizeRequired; } + bool enableDwarfFileDirectoryDefault() const { + return EnableDwarfFileDirectoryDefault; + } + void addInitialFrameState(const MCCFIInstruction &Inst); const std::vector<MCCFIInstruction> &getInitialFrameState() const { diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h index 9d5cb620c9de..80aa97c315da 100644 --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -10,7 +10,6 @@ #define LLVM_MC_MCASSEMBLER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" @@ -18,20 +17,34 @@ #include "llvm/BinaryFormat/MachO.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCDwarf.h" -#include "llvm/MC/MCFixup.h" -#include "llvm/MC/MCFragment.h" #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/SMLoc.h" #include "llvm/Support/VersionTuple.h" +#include <algorithm> #include <cassert> #include <cstddef> #include <cstdint> +#include <memory> #include <string> +#include <tuple> #include <utility> #include <vector> namespace llvm { +class MCBoundaryAlignFragment; +class MCCVDefRangeFragment; +class MCCVInlineLineTableFragment; +class MCDwarfCallFrameFragment; +class MCDwarfLineAddrFragment; +class MCEncodedFragment; +class MCFixup; +class MCLEBFragment; +class MCPseudoProbeAddrFragment; +class MCRelaxableFragment; +class MCSymbolRefExpr; +class raw_ostream; class MCAsmBackend; class MCAsmLayout; class MCContext; diff --git a/llvm/include/llvm/MC/MCCodeView.h b/llvm/include/llvm/MC/MCCodeView.h index 5770f370341d..3d15c4009e43 100644 --- a/llvm/include/llvm/MC/MCCodeView.h +++ b/llvm/include/llvm/MC/MCCodeView.h @@ -13,18 +13,25 @@ #ifndef LLVM_MC_MCCODEVIEW_H #define LLVM_MC_MCCODEVIEW_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCFragment.h" -#include "llvm/MC/MCObjectStreamer.h" #include <map> #include <vector> namespace llvm { +class MCAsmLayout; +class MCCVDefRangeFragment; +class MCCVInlineLineTableFragment; +class MCDataFragment; +class MCFragment; +class MCSection; +class MCSymbol; class MCContext; class MCObjectStreamer; class MCStreamer; -class CodeViewContext; /// Instances of this class represent the information from a /// .cv_loc directive. diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index d2307d692278..a0e18891ed90 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -13,18 +13,15 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCAsmMacro.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCPseudoProbe.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" @@ -44,798 +41,825 @@ namespace llvm { - class CodeViewContext; - class MCAsmInfo; - class MCLabel; - class MCObjectFileInfo; - class MCRegisterInfo; - class MCSection; - class MCSectionCOFF; - class MCSectionELF; - class MCSectionGOFF; - class MCSectionMachO; - class MCSectionWasm; - class MCSectionXCOFF; - class MCStreamer; - class MCSymbol; - class MCSymbolELF; - class MCSymbolWasm; - class MCSymbolXCOFF; - class MDNode; - class SMDiagnostic; - class SMLoc; - class SourceMgr; - - /// Context object for machine code objects. This class owns all of the - /// sections that it creates. - /// - class MCContext { - public: - using SymbolTable = StringMap<MCSymbol *, BumpPtrAllocator &>; - using DiagHandlerTy = - std::function<void(const SMDiagnostic &, bool, const SourceMgr &, - std::vector<const MDNode *> &)>; - enum Environment { IsMachO, IsELF, IsGOFF, IsCOFF, IsWasm, IsXCOFF }; - - private: - Environment Env; - - /// The name of the Segment where Swift5 Reflection Section data will be - /// outputted - StringRef Swift5ReflectionSegmentName; - - /// The triple for this object. - Triple TT; - - /// The SourceMgr for this object, if any. - const SourceMgr *SrcMgr; - - /// The SourceMgr for inline assembly, if any. - std::unique_ptr<SourceMgr> InlineSrcMgr; - std::vector<const MDNode *> LocInfos; - - DiagHandlerTy DiagHandler; - - /// The MCAsmInfo for this target. - const MCAsmInfo *MAI; - - /// The MCRegisterInfo for this target. - const MCRegisterInfo *MRI; - - /// The MCObjectFileInfo for this target. - const MCObjectFileInfo *MOFI; - - /// The MCSubtargetInfo for this target. - const MCSubtargetInfo *MSTI; - - std::unique_ptr<CodeViewContext> CVContext; - - /// Allocator object used for creating machine code objects. - /// - /// We use a bump pointer allocator to avoid the need to track all allocated - /// objects. - BumpPtrAllocator Allocator; - - SpecificBumpPtrAllocator<MCSectionCOFF> COFFAllocator; - SpecificBumpPtrAllocator<MCSectionELF> ELFAllocator; - SpecificBumpPtrAllocator<MCSectionMachO> MachOAllocator; - SpecificBumpPtrAllocator<MCSectionGOFF> GOFFAllocator; - SpecificBumpPtrAllocator<MCSectionWasm> WasmAllocator; - SpecificBumpPtrAllocator<MCSectionXCOFF> XCOFFAllocator; - SpecificBumpPtrAllocator<MCInst> MCInstAllocator; - - /// Bindings of names to symbols. - SymbolTable Symbols; - - /// A mapping from a local label number and an instance count to a symbol. - /// For example, in the assembly - /// 1: - /// 2: - /// 1: - /// We have three labels represented by the pairs (1, 0), (2, 0) and (1, 1) - DenseMap<std::pair<unsigned, unsigned>, MCSymbol *> LocalSymbols; - - /// Keeps tracks of names that were used both for used declared and - /// artificial symbols. The value is "true" if the name has been used for a - /// non-section symbol (there can be at most one of those, plus an unlimited - /// number of section symbols with the same name). - StringMap<bool, BumpPtrAllocator &> UsedNames; - - /// Keeps track of labels that are used in inline assembly. - SymbolTable InlineAsmUsedLabelNames; - - /// The next ID to dole out to an unnamed assembler temporary symbol with - /// a given prefix. - StringMap<unsigned> NextID; - - /// Instances of directional local labels. - DenseMap<unsigned, MCLabel *> Instances; - /// NextInstance() creates the next instance of the directional local label - /// for the LocalLabelVal and adds it to the map if needed. - unsigned NextInstance(unsigned LocalLabelVal); - /// GetInstance() gets the current instance of the directional local label - /// for the LocalLabelVal and adds it to the map if needed. - unsigned GetInstance(unsigned LocalLabelVal); - - /// The file name of the log file from the environment variable - /// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique - /// directive is used or it is an error. - char *SecureLogFile; - /// The stream that gets written to for the .secure_log_unique directive. - std::unique_ptr<raw_fd_ostream> SecureLog; - /// Boolean toggled when .secure_log_unique / .secure_log_reset is seen to - /// catch errors if .secure_log_unique appears twice without - /// .secure_log_reset appearing between them. - bool SecureLogUsed = false; - - /// The compilation directory to use for DW_AT_comp_dir. - SmallString<128> CompilationDir; - - /// Prefix replacement map for source file information. - std::map<const std::string, const std::string> DebugPrefixMap; - - /// The main file name if passed in explicitly. - std::string MainFileName; - - /// The dwarf file and directory tables from the dwarf .file directive. - /// We now emit a line table for each compile unit. To reduce the prologue - /// size of each line table, the files and directories used by each compile - /// unit are separated. - std::map<unsigned, MCDwarfLineTable> MCDwarfLineTablesCUMap; - - /// The current dwarf line information from the last dwarf .loc directive. - MCDwarfLoc CurrentDwarfLoc; - bool DwarfLocSeen = false; - - /// Generate dwarf debugging info for assembly source files. - bool GenDwarfForAssembly = false; - - /// The current dwarf file number when generate dwarf debugging info for - /// assembly source files. - unsigned GenDwarfFileNumber = 0; - - /// Sections for generating the .debug_ranges and .debug_aranges sections. - SetVector<MCSection *> SectionsForRanges; - - /// The information gathered from labels that will have dwarf label - /// entries when generating dwarf assembly source files. - std::vector<MCGenDwarfLabelEntry> MCGenDwarfLabelEntries; - - /// The string to embed in the debug information for the compile unit, if - /// non-empty. - StringRef DwarfDebugFlags; - - /// The string to embed in as the dwarf AT_producer for the compile unit, if - /// non-empty. - StringRef DwarfDebugProducer; - - /// The maximum version of dwarf that we should emit. - uint16_t DwarfVersion = 4; - - /// The format of dwarf that we emit. - dwarf::DwarfFormat DwarfFormat = dwarf::DWARF32; - - /// Honor temporary labels, this is useful for debugging semantic - /// differences between temporary and non-temporary labels (primarily on - /// Darwin). - bool AllowTemporaryLabels = true; - bool UseNamesOnTempLabels = false; - - /// The Compile Unit ID that we are currently processing. - unsigned DwarfCompileUnitID = 0; - - /// A collection of MCPseudoProbe in the current module - MCPseudoProbeTable PseudoProbeTable; - - // Sections are differentiated by the quadruple (section_name, group_name, - // unique_id, link_to_symbol_name). Sections sharing the same quadruple are - // combined into one section. - struct ELFSectionKey { - std::string SectionName; - StringRef GroupName; - StringRef LinkedToName; - unsigned UniqueID; - - ELFSectionKey(StringRef SectionName, StringRef GroupName, - StringRef LinkedToName, unsigned UniqueID) - : SectionName(SectionName), GroupName(GroupName), - LinkedToName(LinkedToName), UniqueID(UniqueID) {} - - bool operator<(const ELFSectionKey &Other) const { - if (SectionName != Other.SectionName) - return SectionName < Other.SectionName; - if (GroupName != Other.GroupName) - return GroupName < Other.GroupName; - if (int O = LinkedToName.compare(Other.LinkedToName)) - return O < 0; - return UniqueID < Other.UniqueID; - } - }; - - struct COFFSectionKey { - std::string SectionName; - StringRef GroupName; - int SelectionKey; - unsigned UniqueID; - - COFFSectionKey(StringRef SectionName, StringRef GroupName, - int SelectionKey, unsigned UniqueID) - : SectionName(SectionName), GroupName(GroupName), - SelectionKey(SelectionKey), UniqueID(UniqueID) {} - - bool operator<(const COFFSectionKey &Other) const { - if (SectionName != Other.SectionName) - return SectionName < Other.SectionName; - if (GroupName != Other.GroupName) - return GroupName < Other.GroupName; - if (SelectionKey != Other.SelectionKey) - return SelectionKey < Other.SelectionKey; - return UniqueID < Other.UniqueID; - } - }; - - struct WasmSectionKey { - std::string SectionName; - StringRef GroupName; - unsigned UniqueID; - - WasmSectionKey(StringRef SectionName, StringRef GroupName, - unsigned UniqueID) - : SectionName(SectionName), GroupName(GroupName), UniqueID(UniqueID) { - } - - bool operator<(const WasmSectionKey &Other) const { - if (SectionName != Other.SectionName) - return SectionName < Other.SectionName; - if (GroupName != Other.GroupName) - return GroupName < Other.GroupName; - return UniqueID < Other.UniqueID; - } - }; - - struct XCOFFSectionKey { - // Section name. - std::string SectionName; - // Section property. - // For csect section, it is storage mapping class. - // For debug section, it is section type flags. - union { - XCOFF::StorageMappingClass MappingClass; - XCOFF::DwarfSectionSubtypeFlags DwarfSubtypeFlags; - }; - bool IsCsect; - - XCOFFSectionKey(StringRef SectionName, - XCOFF::StorageMappingClass MappingClass) - : SectionName(SectionName), MappingClass(MappingClass), - IsCsect(true) {} - - XCOFFSectionKey(StringRef SectionName, - XCOFF::DwarfSectionSubtypeFlags DwarfSubtypeFlags) - : SectionName(SectionName), DwarfSubtypeFlags(DwarfSubtypeFlags), - IsCsect(false) {} - - bool operator<(const XCOFFSectionKey &Other) const { - if (IsCsect && Other.IsCsect) - return std::tie(SectionName, MappingClass) < - std::tie(Other.SectionName, Other.MappingClass); - if (IsCsect != Other.IsCsect) - return IsCsect; - return std::tie(SectionName, DwarfSubtypeFlags) < - std::tie(Other.SectionName, Other.DwarfSubtypeFlags); - } - }; - - StringMap<MCSectionMachO *> MachOUniquingMap; - std::map<ELFSectionKey, MCSectionELF *> ELFUniquingMap; - std::map<COFFSectionKey, MCSectionCOFF *> COFFUniquingMap; - std::map<std::string, MCSectionGOFF *> GOFFUniquingMap; - std::map<WasmSectionKey, MCSectionWasm *> WasmUniquingMap; - std::map<XCOFFSectionKey, MCSectionXCOFF *> XCOFFUniquingMap; - StringMap<bool> RelSecNames; - - SpecificBumpPtrAllocator<MCSubtargetInfo> MCSubtargetAllocator; +class CodeViewContext; +class MCAsmInfo; +class MCInst; +class MCLabel; +class MCObjectFileInfo; +class MCRegisterInfo; +class MCSection; +class MCSectionCOFF; +class MCSectionDXContainer; +class MCSectionELF; +class MCSectionGOFF; +class MCSectionMachO; +class MCSectionSPIRV; +class MCSectionWasm; +class MCSectionXCOFF; +class MCStreamer; +class MCSubtargetInfo; +class MCSymbol; +class MCSymbolELF; +class MCSymbolWasm; +class MCSymbolXCOFF; +class MCTargetOptions; +class MDNode; +template <typename T> class SmallVectorImpl; +class SMDiagnostic; +class SMLoc; +class SourceMgr; +enum class EmitDwarfUnwindType; + +/// Context object for machine code objects. This class owns all of the +/// sections that it creates. +/// +class MCContext { +public: + using SymbolTable = StringMap<MCSymbol *, BumpPtrAllocator &>; + using DiagHandlerTy = + std::function<void(const SMDiagnostic &, bool, const SourceMgr &, + std::vector<const MDNode *> &)>; + enum Environment { + IsMachO, + IsELF, + IsGOFF, + IsCOFF, + IsSPIRV, + IsWasm, + IsXCOFF, + IsDXContainer + }; - /// Do automatic reset in destructor - bool AutoReset; +private: + Environment Env; - MCTargetOptions const *TargetOptions; + /// The name of the Segment where Swift5 Reflection Section data will be + /// outputted + StringRef Swift5ReflectionSegmentName; - bool HadError = false; + /// The triple for this object. + Triple TT; - void reportCommon(SMLoc Loc, - std::function<void(SMDiagnostic &, const SourceMgr *)>); + /// The SourceMgr for this object, if any. + const SourceMgr *SrcMgr; - MCSymbol *createSymbolImpl(const StringMapEntry<bool> *Name, - bool CanBeUnnamed); - MCSymbol *createSymbol(StringRef Name, bool AlwaysAddSuffix, - bool IsTemporary); + /// The SourceMgr for inline assembly, if any. + std::unique_ptr<SourceMgr> InlineSrcMgr; + std::vector<const MDNode *> LocInfos; - MCSymbol *getOrCreateDirectionalLocalSymbol(unsigned LocalLabelVal, - unsigned Instance); + DiagHandlerTy DiagHandler; - MCSectionELF *createELFSectionImpl(StringRef Section, unsigned Type, - unsigned Flags, SectionKind K, - unsigned EntrySize, - const MCSymbolELF *Group, bool IsComdat, - unsigned UniqueID, - const MCSymbolELF *LinkedToSym); + /// The MCAsmInfo for this target. + const MCAsmInfo *MAI; - MCSymbolXCOFF *createXCOFFSymbolImpl(const StringMapEntry<bool> *Name, - bool IsTemporary); + /// The MCRegisterInfo for this target. + const MCRegisterInfo *MRI; - /// Map of currently defined macros. - StringMap<MCAsmMacro> MacroMap; + /// The MCObjectFileInfo for this target. + const MCObjectFileInfo *MOFI; - struct ELFEntrySizeKey { - std::string SectionName; - unsigned Flags; - unsigned EntrySize; + /// The MCSubtargetInfo for this target. + const MCSubtargetInfo *MSTI; - ELFEntrySizeKey(StringRef SectionName, unsigned Flags, unsigned EntrySize) - : SectionName(SectionName), Flags(Flags), EntrySize(EntrySize) {} + std::unique_ptr<CodeViewContext> CVContext; - bool operator<(const ELFEntrySizeKey &Other) const { - if (SectionName != Other.SectionName) - return SectionName < Other.SectionName; - if (Flags != Other.Flags) - return Flags < Other.Flags; - return EntrySize < Other.EntrySize; - } - }; - - // Symbols must be assigned to a section with a compatible entry size and - // flags. This map is used to assign unique IDs to sections to distinguish - // between sections with identical names but incompatible entry sizes and/or - // flags. This can occur when a symbol is explicitly assigned to a section, - // e.g. via __attribute__((section("myname"))). - std::map<ELFEntrySizeKey, unsigned> ELFEntrySizeMap; - - // This set is used to record the generic mergeable section names seen. - // These are sections that are created as mergeable e.g. .debug_str. We need - // to avoid assigning non-mergeable symbols to these sections. It is used - // to prevent non-mergeable symbols being explicitly assigned to mergeable - // sections (e.g. via _attribute_((section("myname")))). - DenseSet<StringRef> ELFSeenGenericMergeableSections; - - public: - explicit MCContext(const Triple &TheTriple, const MCAsmInfo *MAI, - const MCRegisterInfo *MRI, const MCSubtargetInfo *MSTI, - const SourceMgr *Mgr = nullptr, - MCTargetOptions const *TargetOpts = nullptr, - bool DoAutoReset = true, - StringRef Swift5ReflSegmentName = {}); - MCContext(const MCContext &) = delete; - MCContext &operator=(const MCContext &) = delete; - ~MCContext(); - - Environment getObjectFileType() const { return Env; } - - const StringRef &getSwift5ReflectionSegmentName() const { - return Swift5ReflectionSegmentName; + /// Allocator object used for creating machine code objects. + /// + /// We use a bump pointer allocator to avoid the need to track all allocated + /// objects. + BumpPtrAllocator Allocator; + + SpecificBumpPtrAllocator<MCSectionCOFF> COFFAllocator; + SpecificBumpPtrAllocator<MCSectionDXContainer> DXCAllocator; + SpecificBumpPtrAllocator<MCSectionELF> ELFAllocator; + SpecificBumpPtrAllocator<MCSectionMachO> MachOAllocator; + SpecificBumpPtrAllocator<MCSectionGOFF> GOFFAllocator; + SpecificBumpPtrAllocator<MCSectionSPIRV> SPIRVAllocator; + SpecificBumpPtrAllocator<MCSectionWasm> WasmAllocator; + SpecificBumpPtrAllocator<MCSectionXCOFF> XCOFFAllocator; + SpecificBumpPtrAllocator<MCInst> MCInstAllocator; + + /// Bindings of names to symbols. + SymbolTable Symbols; + + /// A mapping from a local label number and an instance count to a symbol. + /// For example, in the assembly + /// 1: + /// 2: + /// 1: + /// We have three labels represented by the pairs (1, 0), (2, 0) and (1, 1) + DenseMap<std::pair<unsigned, unsigned>, MCSymbol *> LocalSymbols; + + /// Keeps tracks of names that were used both for used declared and + /// artificial symbols. The value is "true" if the name has been used for a + /// non-section symbol (there can be at most one of those, plus an unlimited + /// number of section symbols with the same name). + StringMap<bool, BumpPtrAllocator &> UsedNames; + + /// Keeps track of labels that are used in inline assembly. + SymbolTable InlineAsmUsedLabelNames; + + /// The next ID to dole out to an unnamed assembler temporary symbol with + /// a given prefix. + StringMap<unsigned> NextID; + + /// Instances of directional local labels. + DenseMap<unsigned, MCLabel *> Instances; + /// NextInstance() creates the next instance of the directional local label + /// for the LocalLabelVal and adds it to the map if needed. + unsigned NextInstance(unsigned LocalLabelVal); + /// GetInstance() gets the current instance of the directional local label + /// for the LocalLabelVal and adds it to the map if needed. + unsigned GetInstance(unsigned LocalLabelVal); + + /// LLVM_BB_ADDR_MAP version to emit. + uint8_t BBAddrMapVersion = 1; + + /// The file name of the log file from the environment variable + /// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique + /// directive is used or it is an error. + char *SecureLogFile; + /// The stream that gets written to for the .secure_log_unique directive. + std::unique_ptr<raw_fd_ostream> SecureLog; + /// Boolean toggled when .secure_log_unique / .secure_log_reset is seen to + /// catch errors if .secure_log_unique appears twice without + /// .secure_log_reset appearing between them. + bool SecureLogUsed = false; + + /// The compilation directory to use for DW_AT_comp_dir. + SmallString<128> CompilationDir; + + /// Prefix replacement map for source file information. + std::map<const std::string, const std::string> DebugPrefixMap; + + /// The main file name if passed in explicitly. + std::string MainFileName; + + /// The dwarf file and directory tables from the dwarf .file directive. + /// We now emit a line table for each compile unit. To reduce the prologue + /// size of each line table, the files and directories used by each compile + /// unit are separated. + std::map<unsigned, MCDwarfLineTable> MCDwarfLineTablesCUMap; + + /// The current dwarf line information from the last dwarf .loc directive. + MCDwarfLoc CurrentDwarfLoc; + bool DwarfLocSeen = false; + + /// Generate dwarf debugging info for assembly source files. + bool GenDwarfForAssembly = false; + + /// The current dwarf file number when generate dwarf debugging info for + /// assembly source files. + unsigned GenDwarfFileNumber = 0; + + /// Sections for generating the .debug_ranges and .debug_aranges sections. + SetVector<MCSection *> SectionsForRanges; + + /// The information gathered from labels that will have dwarf label + /// entries when generating dwarf assembly source files. + std::vector<MCGenDwarfLabelEntry> MCGenDwarfLabelEntries; + + /// The string to embed in the debug information for the compile unit, if + /// non-empty. + StringRef DwarfDebugFlags; + + /// The string to embed in as the dwarf AT_producer for the compile unit, if + /// non-empty. + StringRef DwarfDebugProducer; + + /// The maximum version of dwarf that we should emit. + uint16_t DwarfVersion = 4; + + /// The format of dwarf that we emit. + dwarf::DwarfFormat DwarfFormat = dwarf::DWARF32; + + /// Honor temporary labels, this is useful for debugging semantic + /// differences between temporary and non-temporary labels (primarily on + /// Darwin). + bool AllowTemporaryLabels = true; + bool UseNamesOnTempLabels = false; + + /// The Compile Unit ID that we are currently processing. + unsigned DwarfCompileUnitID = 0; + + /// A collection of MCPseudoProbe in the current module + MCPseudoProbeTable PseudoProbeTable; + + // Sections are differentiated by the quadruple (section_name, group_name, + // unique_id, link_to_symbol_name). Sections sharing the same quadruple are + // combined into one section. + struct ELFSectionKey { + std::string SectionName; + StringRef GroupName; + StringRef LinkedToName; + unsigned UniqueID; + + ELFSectionKey(StringRef SectionName, StringRef GroupName, + StringRef LinkedToName, unsigned UniqueID) + : SectionName(SectionName), GroupName(GroupName), + LinkedToName(LinkedToName), UniqueID(UniqueID) {} + + bool operator<(const ELFSectionKey &Other) const { + if (SectionName != Other.SectionName) + return SectionName < Other.SectionName; + if (GroupName != Other.GroupName) + return GroupName < Other.GroupName; + if (int O = LinkedToName.compare(Other.LinkedToName)) + return O < 0; + return UniqueID < Other.UniqueID; } - const Triple &getTargetTriple() const { return TT; } - const SourceMgr *getSourceManager() const { return SrcMgr; } + }; - void initInlineSourceManager(); - SourceMgr *getInlineSourceManager() { - return InlineSrcMgr.get(); - } - std::vector<const MDNode *> &getLocInfos() { return LocInfos; } - void setDiagnosticHandler(DiagHandlerTy DiagHandler) { - this->DiagHandler = DiagHandler; + struct COFFSectionKey { + std::string SectionName; + StringRef GroupName; + int SelectionKey; + unsigned UniqueID; + + COFFSectionKey(StringRef SectionName, StringRef GroupName, int SelectionKey, + unsigned UniqueID) + : SectionName(SectionName), GroupName(GroupName), + SelectionKey(SelectionKey), UniqueID(UniqueID) {} + + bool operator<(const COFFSectionKey &Other) const { + if (SectionName != Other.SectionName) + return SectionName < Other.SectionName; + if (GroupName != Other.GroupName) + return GroupName < Other.GroupName; + if (SelectionKey != Other.SelectionKey) + return SelectionKey < Other.SelectionKey; + return UniqueID < Other.UniqueID; } + }; - void setObjectFileInfo(const MCObjectFileInfo *Mofi) { MOFI = Mofi; } - - const MCAsmInfo *getAsmInfo() const { return MAI; } - - const MCRegisterInfo *getRegisterInfo() const { return MRI; } - - const MCObjectFileInfo *getObjectFileInfo() const { return MOFI; } - - const MCSubtargetInfo *getSubtargetInfo() const { return MSTI; } - - CodeViewContext &getCVContext(); - - void setAllowTemporaryLabels(bool Value) { AllowTemporaryLabels = Value; } - void setUseNamesOnTempLabels(bool Value) { UseNamesOnTempLabels = Value; } - - /// \name Module Lifetime Management - /// @{ - - /// reset - return object to right after construction state to prepare - /// to process a new module - void reset(); - - /// @} - - /// \name McInst Management - - /// Create and return a new MC instruction. - MCInst *createMCInst(); - - /// \name Symbol Management - /// @{ - - /// Create and return a new linker temporary symbol with a unique but - /// unspecified name. - MCSymbol *createLinkerPrivateTempSymbol(); - - /// Create a temporary symbol with a unique name. The name will be omitted - /// in the symbol table if UseNamesOnTempLabels is false (default except - /// MCAsmStreamer). The overload without Name uses an unspecified name. - MCSymbol *createTempSymbol(); - MCSymbol *createTempSymbol(const Twine &Name, bool AlwaysAddSuffix = true); - - /// Create a temporary symbol with a unique name whose name cannot be - /// omitted in the symbol table. This is rarely used. - MCSymbol *createNamedTempSymbol(); - MCSymbol *createNamedTempSymbol(const Twine &Name); - - /// Create the definition of a directional local symbol for numbered label - /// (used for "1:" definitions). - MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal); - - /// Create and return a directional local symbol for numbered label (used - /// for "1b" or 1f" references). - MCSymbol *getDirectionalLocalSymbol(unsigned LocalLabelVal, bool Before); - - /// Lookup the symbol inside with the specified \p Name. If it exists, - /// return it. If not, create a forward reference and return it. - /// - /// \param Name - The symbol name, which must be unique across all symbols. - MCSymbol *getOrCreateSymbol(const Twine &Name); + struct WasmSectionKey { + std::string SectionName; + StringRef GroupName; + unsigned UniqueID; + + WasmSectionKey(StringRef SectionName, StringRef GroupName, + unsigned UniqueID) + : SectionName(SectionName), GroupName(GroupName), UniqueID(UniqueID) {} + + bool operator<(const WasmSectionKey &Other) const { + if (SectionName != Other.SectionName) + return SectionName < Other.SectionName; + if (GroupName != Other.GroupName) + return GroupName < Other.GroupName; + return UniqueID < Other.UniqueID; + } + }; - /// Gets a symbol that will be defined to the final stack offset of a local - /// variable after codegen. - /// - /// \param Idx - The index of a local variable passed to \@llvm.localescape. - MCSymbol *getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx); + struct XCOFFSectionKey { + // Section name. + std::string SectionName; + // Section property. + // For csect section, it is storage mapping class. + // For debug section, it is section type flags. + union { + XCOFF::StorageMappingClass MappingClass; + XCOFF::DwarfSectionSubtypeFlags DwarfSubtypeFlags; + }; + bool IsCsect; + + XCOFFSectionKey(StringRef SectionName, + XCOFF::StorageMappingClass MappingClass) + : SectionName(SectionName), MappingClass(MappingClass), IsCsect(true) {} + + XCOFFSectionKey(StringRef SectionName, + XCOFF::DwarfSectionSubtypeFlags DwarfSubtypeFlags) + : SectionName(SectionName), DwarfSubtypeFlags(DwarfSubtypeFlags), + IsCsect(false) {} + + bool operator<(const XCOFFSectionKey &Other) const { + if (IsCsect && Other.IsCsect) + return std::tie(SectionName, MappingClass) < + std::tie(Other.SectionName, Other.MappingClass); + if (IsCsect != Other.IsCsect) + return IsCsect; + return std::tie(SectionName, DwarfSubtypeFlags) < + std::tie(Other.SectionName, Other.DwarfSubtypeFlags); + } + }; - MCSymbol *getOrCreateParentFrameOffsetSymbol(StringRef FuncName); + StringMap<MCSectionMachO *> MachOUniquingMap; + std::map<ELFSectionKey, MCSectionELF *> ELFUniquingMap; + std::map<COFFSectionKey, MCSectionCOFF *> COFFUniquingMap; + std::map<std::string, MCSectionGOFF *> GOFFUniquingMap; + std::map<WasmSectionKey, MCSectionWasm *> WasmUniquingMap; + std::map<XCOFFSectionKey, MCSectionXCOFF *> XCOFFUniquingMap; + StringMap<MCSectionDXContainer *> DXCUniquingMap; + StringMap<bool> RelSecNames; - MCSymbol *getOrCreateLSDASymbol(StringRef FuncName); + SpecificBumpPtrAllocator<MCSubtargetInfo> MCSubtargetAllocator; - /// Get the symbol for \p Name, or null. - MCSymbol *lookupSymbol(const Twine &Name) const; + /// Do automatic reset in destructor + bool AutoReset; - /// Set value for a symbol. - void setSymbolValue(MCStreamer &Streamer, StringRef Sym, uint64_t Val); + MCTargetOptions const *TargetOptions; - /// getSymbols - Get a reference for the symbol table for clients that - /// want to, for example, iterate over all symbols. 'const' because we - /// still want any modifications to the table itself to use the MCContext - /// APIs. - const SymbolTable &getSymbols() const { return Symbols; } + bool HadError = false; - /// isInlineAsmLabel - Return true if the name is a label referenced in - /// inline assembly. - MCSymbol *getInlineAsmLabel(StringRef Name) const { - return InlineAsmUsedLabelNames.lookup(Name); - } + void reportCommon(SMLoc Loc, + std::function<void(SMDiagnostic &, const SourceMgr *)>); - /// registerInlineAsmLabel - Records that the name is a label referenced in - /// inline assembly. - void registerInlineAsmLabel(MCSymbol *Sym); + MCSymbol *createSymbolImpl(const StringMapEntry<bool> *Name, + bool CanBeUnnamed); + MCSymbol *createSymbol(StringRef Name, bool AlwaysAddSuffix, + bool IsTemporary); - /// @} + MCSymbol *getOrCreateDirectionalLocalSymbol(unsigned LocalLabelVal, + unsigned Instance); - /// \name Section Management - /// @{ + MCSectionELF *createELFSectionImpl(StringRef Section, unsigned Type, + unsigned Flags, SectionKind K, + unsigned EntrySize, + const MCSymbolELF *Group, bool IsComdat, + unsigned UniqueID, + const MCSymbolELF *LinkedToSym); - enum : unsigned { - /// Pass this value as the UniqueID during section creation to get the - /// generic section with the given name and characteristics. The usual - /// sections such as .text use this ID. - GenericSectionID = ~0U - }; + MCSymbolXCOFF *createXCOFFSymbolImpl(const StringMapEntry<bool> *Name, + bool IsTemporary); - /// Return the MCSection for the specified mach-o section. This requires - /// the operands to be valid. - MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section, - unsigned TypeAndAttributes, - unsigned Reserved2, SectionKind K, - const char *BeginSymName = nullptr); - - MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section, - unsigned TypeAndAttributes, SectionKind K, - const char *BeginSymName = nullptr) { - return getMachOSection(Segment, Section, TypeAndAttributes, 0, K, - BeginSymName); - } + /// Map of currently defined macros. + StringMap<MCAsmMacro> MacroMap; - MCSectionELF *getELFSection(const Twine &Section, unsigned Type, - unsigned Flags) { - return getELFSection(Section, Type, Flags, 0, "", false); - } + struct ELFEntrySizeKey { + std::string SectionName; + unsigned Flags; + unsigned EntrySize; - MCSectionELF *getELFSection(const Twine &Section, unsigned Type, - unsigned Flags, unsigned EntrySize) { - return getELFSection(Section, Type, Flags, EntrySize, "", false, - MCSection::NonUniqueID, nullptr); - } + ELFEntrySizeKey(StringRef SectionName, unsigned Flags, unsigned EntrySize) + : SectionName(SectionName), Flags(Flags), EntrySize(EntrySize) {} - MCSectionELF *getELFSection(const Twine &Section, unsigned Type, - unsigned Flags, unsigned EntrySize, - const Twine &Group, bool IsComdat) { - return getELFSection(Section, Type, Flags, EntrySize, Group, IsComdat, - MCSection::NonUniqueID, nullptr); + bool operator<(const ELFEntrySizeKey &Other) const { + if (SectionName != Other.SectionName) + return SectionName < Other.SectionName; + if (Flags != Other.Flags) + return Flags < Other.Flags; + return EntrySize < Other.EntrySize; } + }; - MCSectionELF *getELFSection(const Twine &Section, unsigned Type, - unsigned Flags, unsigned EntrySize, - const Twine &Group, bool IsComdat, - unsigned UniqueID, - const MCSymbolELF *LinkedToSym); - - MCSectionELF *getELFSection(const Twine &Section, unsigned Type, - unsigned Flags, unsigned EntrySize, - const MCSymbolELF *Group, bool IsComdat, - unsigned UniqueID, - const MCSymbolELF *LinkedToSym); - - /// Get a section with the provided group identifier. This section is - /// named by concatenating \p Prefix with '.' then \p Suffix. The \p Type - /// describes the type of the section and \p Flags are used to further - /// configure this named section. - MCSectionELF *getELFNamedSection(const Twine &Prefix, const Twine &Suffix, - unsigned Type, unsigned Flags, - unsigned EntrySize = 0); - - MCSectionELF *createELFRelSection(const Twine &Name, unsigned Type, - unsigned Flags, unsigned EntrySize, - const MCSymbolELF *Group, - const MCSectionELF *RelInfoSection); - - void renameELFSection(MCSectionELF *Section, StringRef Name); + // Symbols must be assigned to a section with a compatible entry size and + // flags. This map is used to assign unique IDs to sections to distinguish + // between sections with identical names but incompatible entry sizes and/or + // flags. This can occur when a symbol is explicitly assigned to a section, + // e.g. via __attribute__((section("myname"))). + std::map<ELFEntrySizeKey, unsigned> ELFEntrySizeMap; - MCSectionELF *createELFGroupSection(const MCSymbolELF *Group, - bool IsComdat); + // This set is used to record the generic mergeable section names seen. + // These are sections that are created as mergeable e.g. .debug_str. We need + // to avoid assigning non-mergeable symbols to these sections. It is used + // to prevent non-mergeable symbols being explicitly assigned to mergeable + // sections (e.g. via _attribute_((section("myname")))). + DenseSet<StringRef> ELFSeenGenericMergeableSections; - void recordELFMergeableSectionInfo(StringRef SectionName, unsigned Flags, - unsigned UniqueID, unsigned EntrySize); +public: + explicit MCContext(const Triple &TheTriple, const MCAsmInfo *MAI, + const MCRegisterInfo *MRI, const MCSubtargetInfo *MSTI, + const SourceMgr *Mgr = nullptr, + MCTargetOptions const *TargetOpts = nullptr, + bool DoAutoReset = true, + StringRef Swift5ReflSegmentName = {}); + MCContext(const MCContext &) = delete; + MCContext &operator=(const MCContext &) = delete; + ~MCContext(); - bool isELFImplicitMergeableSectionNamePrefix(StringRef Name); + Environment getObjectFileType() const { return Env; } - bool isELFGenericMergeableSection(StringRef Name); + const StringRef &getSwift5ReflectionSegmentName() const { + return Swift5ReflectionSegmentName; + } + const Triple &getTargetTriple() const { return TT; } + const SourceMgr *getSourceManager() const { return SrcMgr; } - /// Return the unique ID of the section with the given name, flags and entry - /// size, if it exists. - Optional<unsigned> getELFUniqueIDForEntsize(StringRef SectionName, - unsigned Flags, - unsigned EntrySize); + void initInlineSourceManager(); + SourceMgr *getInlineSourceManager() { return InlineSrcMgr.get(); } + std::vector<const MDNode *> &getLocInfos() { return LocInfos; } + void setDiagnosticHandler(DiagHandlerTy DiagHandler) { + this->DiagHandler = DiagHandler; + } - MCSectionGOFF *getGOFFSection(StringRef Section, SectionKind Kind); + void setObjectFileInfo(const MCObjectFileInfo *Mofi) { MOFI = Mofi; } - MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics, - SectionKind Kind, StringRef COMDATSymName, - int Selection, - unsigned UniqueID = GenericSectionID, - const char *BeginSymName = nullptr); + const MCAsmInfo *getAsmInfo() const { return MAI; } - MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics, - SectionKind Kind, - const char *BeginSymName = nullptr); - - /// Gets or creates a section equivalent to Sec that is associated with the - /// section containing KeySym. For example, to create a debug info section - /// associated with an inline function, pass the normal debug info section - /// as Sec and the function symbol as KeySym. - MCSectionCOFF * - getAssociativeCOFFSection(MCSectionCOFF *Sec, const MCSymbol *KeySym, - unsigned UniqueID = GenericSectionID); - - MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, - unsigned Flags = 0) { - return getWasmSection(Section, K, Flags, nullptr); - } + const MCRegisterInfo *getRegisterInfo() const { return MRI; } - MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, - unsigned Flags, const char *BeginSymName) { - return getWasmSection(Section, K, Flags, "", ~0, BeginSymName); - } + const MCObjectFileInfo *getObjectFileInfo() const { return MOFI; } - MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, - unsigned Flags, const Twine &Group, - unsigned UniqueID) { - return getWasmSection(Section, K, Flags, Group, UniqueID, nullptr); - } + const MCSubtargetInfo *getSubtargetInfo() const { return MSTI; } - MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, - unsigned Flags, const Twine &Group, - unsigned UniqueID, const char *BeginSymName); + CodeViewContext &getCVContext(); - MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, - unsigned Flags, const MCSymbolWasm *Group, - unsigned UniqueID, const char *BeginSymName); + void setAllowTemporaryLabels(bool Value) { AllowTemporaryLabels = Value; } + void setUseNamesOnTempLabels(bool Value) { UseNamesOnTempLabels = Value; } - MCSectionXCOFF *getXCOFFSection( - StringRef Section, SectionKind K, - Optional<XCOFF::CsectProperties> CsectProp = None, - bool MultiSymbolsAllowed = false, const char *BeginSymName = nullptr, - Optional<XCOFF::DwarfSectionSubtypeFlags> DwarfSubtypeFlags = None); + /// \name Module Lifetime Management + /// @{ - // Create and save a copy of STI and return a reference to the copy. - MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI); + /// reset - return object to right after construction state to prepare + /// to process a new module + void reset(); - /// @} + /// @} - /// \name Dwarf Management - /// @{ + /// \name McInst Management - /// Get the compilation directory for DW_AT_comp_dir - /// The compilation directory should be set with \c setCompilationDir before - /// calling this function. If it is unset, an empty string will be returned. - StringRef getCompilationDir() const { return CompilationDir; } + /// Create and return a new MC instruction. + MCInst *createMCInst(); - /// Set the compilation directory for DW_AT_comp_dir - void setCompilationDir(StringRef S) { CompilationDir = S.str(); } + /// \name Symbol Management + /// @{ - /// Add an entry to the debug prefix map. - void addDebugPrefixMapEntry(const std::string &From, const std::string &To); + /// Create and return a new linker temporary symbol with a unique but + /// unspecified name. + MCSymbol *createLinkerPrivateTempSymbol(); - // Remaps all debug directory paths in-place as per the debug prefix map. - void RemapDebugPaths(); + /// Create a temporary symbol with a unique name. The name will be omitted + /// in the symbol table if UseNamesOnTempLabels is false (default except + /// MCAsmStreamer). The overload without Name uses an unspecified name. + MCSymbol *createTempSymbol(); + MCSymbol *createTempSymbol(const Twine &Name, bool AlwaysAddSuffix = true); - /// Get the main file name for use in error messages and debug - /// info. This can be set to ensure we've got the correct file name - /// after preprocessing or for -save-temps. - const std::string &getMainFileName() const { return MainFileName; } + /// Create a temporary symbol with a unique name whose name cannot be + /// omitted in the symbol table. This is rarely used. + MCSymbol *createNamedTempSymbol(); + MCSymbol *createNamedTempSymbol(const Twine &Name); - /// Set the main file name and override the default. - void setMainFileName(StringRef S) { MainFileName = std::string(S); } + /// Create the definition of a directional local symbol for numbered label + /// (used for "1:" definitions). + MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal); - /// Creates an entry in the dwarf file and directory tables. - Expected<unsigned> getDwarfFile(StringRef Directory, StringRef FileName, - unsigned FileNumber, - Optional<MD5::MD5Result> Checksum, - Optional<StringRef> Source, unsigned CUID); + /// Create and return a directional local symbol for numbered label (used + /// for "1b" or 1f" references). + MCSymbol *getDirectionalLocalSymbol(unsigned LocalLabelVal, bool Before); - bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0); + /// Lookup the symbol inside with the specified \p Name. If it exists, + /// return it. If not, create a forward reference and return it. + /// + /// \param Name - The symbol name, which must be unique across all symbols. + MCSymbol *getOrCreateSymbol(const Twine &Name); - const std::map<unsigned, MCDwarfLineTable> &getMCDwarfLineTables() const { - return MCDwarfLineTablesCUMap; - } + /// Gets a symbol that will be defined to the final stack offset of a local + /// variable after codegen. + /// + /// \param Idx - The index of a local variable passed to \@llvm.localescape. + MCSymbol *getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx); - MCDwarfLineTable &getMCDwarfLineTable(unsigned CUID) { - return MCDwarfLineTablesCUMap[CUID]; - } + MCSymbol *getOrCreateParentFrameOffsetSymbol(StringRef FuncName); - const MCDwarfLineTable &getMCDwarfLineTable(unsigned CUID) const { - auto I = MCDwarfLineTablesCUMap.find(CUID); - assert(I != MCDwarfLineTablesCUMap.end()); - return I->second; - } + MCSymbol *getOrCreateLSDASymbol(StringRef FuncName); - const SmallVectorImpl<MCDwarfFile> &getMCDwarfFiles(unsigned CUID = 0) { - return getMCDwarfLineTable(CUID).getMCDwarfFiles(); - } + /// Get the symbol for \p Name, or null. + MCSymbol *lookupSymbol(const Twine &Name) const; - const SmallVectorImpl<std::string> &getMCDwarfDirs(unsigned CUID = 0) { - return getMCDwarfLineTable(CUID).getMCDwarfDirs(); - } + /// Set value for a symbol. + void setSymbolValue(MCStreamer &Streamer, StringRef Sym, uint64_t Val); - unsigned getDwarfCompileUnitID() { return DwarfCompileUnitID; } + /// getSymbols - Get a reference for the symbol table for clients that + /// want to, for example, iterate over all symbols. 'const' because we + /// still want any modifications to the table itself to use the MCContext + /// APIs. + const SymbolTable &getSymbols() const { return Symbols; } - void setDwarfCompileUnitID(unsigned CUIndex) { - DwarfCompileUnitID = CUIndex; - } + /// isInlineAsmLabel - Return true if the name is a label referenced in + /// inline assembly. + MCSymbol *getInlineAsmLabel(StringRef Name) const { + return InlineAsmUsedLabelNames.lookup(Name); + } - /// Specifies the "root" file and directory of the compilation unit. - /// These are "file 0" and "directory 0" in DWARF v5. - void setMCLineTableRootFile(unsigned CUID, StringRef CompilationDir, - StringRef Filename, - Optional<MD5::MD5Result> Checksum, - Optional<StringRef> Source) { - getMCDwarfLineTable(CUID).setRootFile(CompilationDir, Filename, Checksum, - Source); - } + /// registerInlineAsmLabel - Records that the name is a label referenced in + /// inline assembly. + void registerInlineAsmLabel(MCSymbol *Sym); - /// Reports whether MD5 checksum usage is consistent (all-or-none). - bool isDwarfMD5UsageConsistent(unsigned CUID) const { - return getMCDwarfLineTable(CUID).isMD5UsageConsistent(); - } + /// @} - /// Saves the information from the currently parsed dwarf .loc directive - /// and sets DwarfLocSeen. When the next instruction is assembled an entry - /// in the line number table with this information and the address of the - /// instruction will be created. - void setCurrentDwarfLoc(unsigned FileNum, unsigned Line, unsigned Column, - unsigned Flags, unsigned Isa, - unsigned Discriminator) { - CurrentDwarfLoc.setFileNum(FileNum); - CurrentDwarfLoc.setLine(Line); - CurrentDwarfLoc.setColumn(Column); - CurrentDwarfLoc.setFlags(Flags); - CurrentDwarfLoc.setIsa(Isa); - CurrentDwarfLoc.setDiscriminator(Discriminator); - DwarfLocSeen = true; - } + /// \name Section Management + /// @{ - void clearDwarfLocSeen() { DwarfLocSeen = false; } + enum : unsigned { + /// Pass this value as the UniqueID during section creation to get the + /// generic section with the given name and characteristics. The usual + /// sections such as .text use this ID. + GenericSectionID = ~0U + }; - bool getDwarfLocSeen() { return DwarfLocSeen; } - const MCDwarfLoc &getCurrentDwarfLoc() { return CurrentDwarfLoc; } + /// Return the MCSection for the specified mach-o section. This requires + /// the operands to be valid. + MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section, + unsigned TypeAndAttributes, + unsigned Reserved2, SectionKind K, + const char *BeginSymName = nullptr); - bool getGenDwarfForAssembly() { return GenDwarfForAssembly; } - void setGenDwarfForAssembly(bool Value) { GenDwarfForAssembly = Value; } - unsigned getGenDwarfFileNumber() { return GenDwarfFileNumber; } + MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section, + unsigned TypeAndAttributes, SectionKind K, + const char *BeginSymName = nullptr) { + return getMachOSection(Segment, Section, TypeAndAttributes, 0, K, + BeginSymName); + } + + MCSectionELF *getELFSection(const Twine &Section, unsigned Type, + unsigned Flags) { + return getELFSection(Section, Type, Flags, 0, "", false); + } + + MCSectionELF *getELFSection(const Twine &Section, unsigned Type, + unsigned Flags, unsigned EntrySize) { + return getELFSection(Section, Type, Flags, EntrySize, "", false, + MCSection::NonUniqueID, nullptr); + } + + MCSectionELF *getELFSection(const Twine &Section, unsigned Type, + unsigned Flags, unsigned EntrySize, + const Twine &Group, bool IsComdat) { + return getELFSection(Section, Type, Flags, EntrySize, Group, IsComdat, + MCSection::NonUniqueID, nullptr); + } + + MCSectionELF *getELFSection(const Twine &Section, unsigned Type, + unsigned Flags, unsigned EntrySize, + const Twine &Group, bool IsComdat, + unsigned UniqueID, + const MCSymbolELF *LinkedToSym); + + MCSectionELF *getELFSection(const Twine &Section, unsigned Type, + unsigned Flags, unsigned EntrySize, + const MCSymbolELF *Group, bool IsComdat, + unsigned UniqueID, + const MCSymbolELF *LinkedToSym); + + /// Get a section with the provided group identifier. This section is + /// named by concatenating \p Prefix with '.' then \p Suffix. The \p Type + /// describes the type of the section and \p Flags are used to further + /// configure this named section. + MCSectionELF *getELFNamedSection(const Twine &Prefix, const Twine &Suffix, + unsigned Type, unsigned Flags, + unsigned EntrySize = 0); + + MCSectionELF *createELFRelSection(const Twine &Name, unsigned Type, + unsigned Flags, unsigned EntrySize, + const MCSymbolELF *Group, + const MCSectionELF *RelInfoSection); + + void renameELFSection(MCSectionELF *Section, StringRef Name); + + MCSectionELF *createELFGroupSection(const MCSymbolELF *Group, bool IsComdat); + + void recordELFMergeableSectionInfo(StringRef SectionName, unsigned Flags, + unsigned UniqueID, unsigned EntrySize); + + bool isELFImplicitMergeableSectionNamePrefix(StringRef Name); + + bool isELFGenericMergeableSection(StringRef Name); + + /// Return the unique ID of the section with the given name, flags and entry + /// size, if it exists. + Optional<unsigned> getELFUniqueIDForEntsize(StringRef SectionName, + unsigned Flags, + unsigned EntrySize); + + MCSectionGOFF *getGOFFSection(StringRef Section, SectionKind Kind, + MCSection *Parent, const MCExpr *SubsectionId); + + MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics, + SectionKind Kind, StringRef COMDATSymName, + int Selection, + unsigned UniqueID = GenericSectionID, + const char *BeginSymName = nullptr); + + MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics, + SectionKind Kind, + const char *BeginSymName = nullptr); + + /// Gets or creates a section equivalent to Sec that is associated with the + /// section containing KeySym. For example, to create a debug info section + /// associated with an inline function, pass the normal debug info section + /// as Sec and the function symbol as KeySym. + MCSectionCOFF * + getAssociativeCOFFSection(MCSectionCOFF *Sec, const MCSymbol *KeySym, + unsigned UniqueID = GenericSectionID); + + MCSectionSPIRV *getSPIRVSection(); + + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + unsigned Flags = 0) { + return getWasmSection(Section, K, Flags, nullptr); + } + + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + unsigned Flags, const char *BeginSymName) { + return getWasmSection(Section, K, Flags, "", ~0, BeginSymName); + } + + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + unsigned Flags, const Twine &Group, + unsigned UniqueID) { + return getWasmSection(Section, K, Flags, Group, UniqueID, nullptr); + } + + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + unsigned Flags, const Twine &Group, + unsigned UniqueID, const char *BeginSymName); + + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + unsigned Flags, const MCSymbolWasm *Group, + unsigned UniqueID, const char *BeginSymName); + + /// Get the section for the provided Section name + MCSectionDXContainer *getDXContainerSection(StringRef Section, SectionKind K); + + bool hasXCOFFSection(StringRef Section, + XCOFF::CsectProperties CsectProp) const; + + MCSectionXCOFF *getXCOFFSection( + StringRef Section, SectionKind K, + Optional<XCOFF::CsectProperties> CsectProp = None, + bool MultiSymbolsAllowed = false, const char *BeginSymName = nullptr, + Optional<XCOFF::DwarfSectionSubtypeFlags> DwarfSubtypeFlags = None); + + // Create and save a copy of STI and return a reference to the copy. + MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI); + + uint8_t getBBAddrMapVersion() const { return BBAddrMapVersion; } + + /// @} + + /// \name Dwarf Management + /// @{ + + /// Get the compilation directory for DW_AT_comp_dir + /// The compilation directory should be set with \c setCompilationDir before + /// calling this function. If it is unset, an empty string will be returned. + StringRef getCompilationDir() const { return CompilationDir; } + + /// Set the compilation directory for DW_AT_comp_dir + void setCompilationDir(StringRef S) { CompilationDir = S.str(); } + + /// Add an entry to the debug prefix map. + void addDebugPrefixMapEntry(const std::string &From, const std::string &To); + + // Remaps all debug directory paths in-place as per the debug prefix map. + void RemapDebugPaths(); + + /// Get the main file name for use in error messages and debug + /// info. This can be set to ensure we've got the correct file name + /// after preprocessing or for -save-temps. + const std::string &getMainFileName() const { return MainFileName; } + + /// Set the main file name and override the default. + void setMainFileName(StringRef S) { MainFileName = std::string(S); } + + /// Creates an entry in the dwarf file and directory tables. + Expected<unsigned> getDwarfFile(StringRef Directory, StringRef FileName, + unsigned FileNumber, + Optional<MD5::MD5Result> Checksum, + Optional<StringRef> Source, unsigned CUID); + + bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0); + + const std::map<unsigned, MCDwarfLineTable> &getMCDwarfLineTables() const { + return MCDwarfLineTablesCUMap; + } + + MCDwarfLineTable &getMCDwarfLineTable(unsigned CUID) { + return MCDwarfLineTablesCUMap[CUID]; + } + + const MCDwarfLineTable &getMCDwarfLineTable(unsigned CUID) const { + auto I = MCDwarfLineTablesCUMap.find(CUID); + assert(I != MCDwarfLineTablesCUMap.end()); + return I->second; + } + + const SmallVectorImpl<MCDwarfFile> &getMCDwarfFiles(unsigned CUID = 0) { + return getMCDwarfLineTable(CUID).getMCDwarfFiles(); + } + + const SmallVectorImpl<std::string> &getMCDwarfDirs(unsigned CUID = 0) { + return getMCDwarfLineTable(CUID).getMCDwarfDirs(); + } - void setGenDwarfFileNumber(unsigned FileNumber) { - GenDwarfFileNumber = FileNumber; - } + unsigned getDwarfCompileUnitID() { return DwarfCompileUnitID; } - /// Specifies information about the "root file" for assembler clients - /// (e.g., llvm-mc). Assumes compilation dir etc. have been set up. - void setGenDwarfRootFile(StringRef FileName, StringRef Buffer); + void setDwarfCompileUnitID(unsigned CUIndex) { DwarfCompileUnitID = CUIndex; } + + /// Specifies the "root" file and directory of the compilation unit. + /// These are "file 0" and "directory 0" in DWARF v5. + void setMCLineTableRootFile(unsigned CUID, StringRef CompilationDir, + StringRef Filename, + Optional<MD5::MD5Result> Checksum, + Optional<StringRef> Source) { + getMCDwarfLineTable(CUID).setRootFile(CompilationDir, Filename, Checksum, + Source); + } + + /// Reports whether MD5 checksum usage is consistent (all-or-none). + bool isDwarfMD5UsageConsistent(unsigned CUID) const { + return getMCDwarfLineTable(CUID).isMD5UsageConsistent(); + } - const SetVector<MCSection *> &getGenDwarfSectionSyms() { - return SectionsForRanges; - } + /// Saves the information from the currently parsed dwarf .loc directive + /// and sets DwarfLocSeen. When the next instruction is assembled an entry + /// in the line number table with this information and the address of the + /// instruction will be created. + void setCurrentDwarfLoc(unsigned FileNum, unsigned Line, unsigned Column, + unsigned Flags, unsigned Isa, + unsigned Discriminator) { + CurrentDwarfLoc.setFileNum(FileNum); + CurrentDwarfLoc.setLine(Line); + CurrentDwarfLoc.setColumn(Column); + CurrentDwarfLoc.setFlags(Flags); + CurrentDwarfLoc.setIsa(Isa); + CurrentDwarfLoc.setDiscriminator(Discriminator); + DwarfLocSeen = true; + } - bool addGenDwarfSection(MCSection *Sec) { - return SectionsForRanges.insert(Sec); - } + void clearDwarfLocSeen() { DwarfLocSeen = false; } - void finalizeDwarfSections(MCStreamer &MCOS); + bool getDwarfLocSeen() { return DwarfLocSeen; } + const MCDwarfLoc &getCurrentDwarfLoc() { return CurrentDwarfLoc; } - const std::vector<MCGenDwarfLabelEntry> &getMCGenDwarfLabelEntries() const { - return MCGenDwarfLabelEntries; - } + bool getGenDwarfForAssembly() { return GenDwarfForAssembly; } + void setGenDwarfForAssembly(bool Value) { GenDwarfForAssembly = Value; } + unsigned getGenDwarfFileNumber() { return GenDwarfFileNumber; } + EmitDwarfUnwindType emitDwarfUnwindInfo() const; - void addMCGenDwarfLabelEntry(const MCGenDwarfLabelEntry &E) { - MCGenDwarfLabelEntries.push_back(E); - } + void setGenDwarfFileNumber(unsigned FileNumber) { + GenDwarfFileNumber = FileNumber; + } - void setDwarfDebugFlags(StringRef S) { DwarfDebugFlags = S; } - StringRef getDwarfDebugFlags() { return DwarfDebugFlags; } + /// Specifies information about the "root file" for assembler clients + /// (e.g., llvm-mc). Assumes compilation dir etc. have been set up. + void setGenDwarfRootFile(StringRef FileName, StringRef Buffer); - void setDwarfDebugProducer(StringRef S) { DwarfDebugProducer = S; } - StringRef getDwarfDebugProducer() { return DwarfDebugProducer; } + const SetVector<MCSection *> &getGenDwarfSectionSyms() { + return SectionsForRanges; + } - void setDwarfFormat(dwarf::DwarfFormat f) { DwarfFormat = f; } - dwarf::DwarfFormat getDwarfFormat() const { return DwarfFormat; } + bool addGenDwarfSection(MCSection *Sec) { + return SectionsForRanges.insert(Sec); + } - void setDwarfVersion(uint16_t v) { DwarfVersion = v; } - uint16_t getDwarfVersion() const { return DwarfVersion; } + void finalizeDwarfSections(MCStreamer &MCOS); + + const std::vector<MCGenDwarfLabelEntry> &getMCGenDwarfLabelEntries() const { + return MCGenDwarfLabelEntries; + } - /// @} + void addMCGenDwarfLabelEntry(const MCGenDwarfLabelEntry &E) { + MCGenDwarfLabelEntries.push_back(E); + } + + void setDwarfDebugFlags(StringRef S) { DwarfDebugFlags = S; } + StringRef getDwarfDebugFlags() { return DwarfDebugFlags; } - char *getSecureLogFile() { return SecureLogFile; } - raw_fd_ostream *getSecureLog() { return SecureLog.get(); } + void setDwarfDebugProducer(StringRef S) { DwarfDebugProducer = S; } + StringRef getDwarfDebugProducer() { return DwarfDebugProducer; } - void setSecureLog(std::unique_ptr<raw_fd_ostream> Value) { - SecureLog = std::move(Value); - } + void setDwarfFormat(dwarf::DwarfFormat f) { DwarfFormat = f; } + dwarf::DwarfFormat getDwarfFormat() const { return DwarfFormat; } + + void setDwarfVersion(uint16_t v) { DwarfVersion = v; } + uint16_t getDwarfVersion() const { return DwarfVersion; } + + /// @} - bool getSecureLogUsed() { return SecureLogUsed; } - void setSecureLogUsed(bool Value) { SecureLogUsed = Value; } + char *getSecureLogFile() { return SecureLogFile; } + raw_fd_ostream *getSecureLog() { return SecureLog.get(); } + + void setSecureLog(std::unique_ptr<raw_fd_ostream> Value) { + SecureLog = std::move(Value); + } - void *allocate(unsigned Size, unsigned Align = 8) { - return Allocator.Allocate(Size, Align); - } + bool getSecureLogUsed() { return SecureLogUsed; } + void setSecureLogUsed(bool Value) { SecureLogUsed = Value; } - void deallocate(void *Ptr) {} + void *allocate(unsigned Size, unsigned Align = 8) { + return Allocator.Allocate(Size, Align); + } - bool hadError() { return HadError; } - void diagnose(const SMDiagnostic &SMD); - void reportError(SMLoc L, const Twine &Msg); - void reportWarning(SMLoc L, const Twine &Msg); + void deallocate(void *Ptr) {} - const MCAsmMacro *lookupMacro(StringRef Name) { - StringMap<MCAsmMacro>::iterator I = MacroMap.find(Name); - return (I == MacroMap.end()) ? nullptr : &I->getValue(); - } + bool hadError() { return HadError; } + void diagnose(const SMDiagnostic &SMD); + void reportError(SMLoc L, const Twine &Msg); + void reportWarning(SMLoc L, const Twine &Msg); + + const MCAsmMacro *lookupMacro(StringRef Name) { + StringMap<MCAsmMacro>::iterator I = MacroMap.find(Name); + return (I == MacroMap.end()) ? nullptr : &I->getValue(); + } - void defineMacro(StringRef Name, MCAsmMacro Macro) { - MacroMap.insert(std::make_pair(Name, std::move(Macro))); - } + void defineMacro(StringRef Name, MCAsmMacro Macro) { + MacroMap.insert(std::make_pair(Name, std::move(Macro))); + } - void undefineMacro(StringRef Name) { MacroMap.erase(Name); } + void undefineMacro(StringRef Name) { MacroMap.erase(Name); } - MCPseudoProbeTable &getMCPseudoProbeTable() { return PseudoProbeTable; } - }; + MCPseudoProbeTable &getMCPseudoProbeTable() { return PseudoProbeTable; } +}; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCDXContainerStreamer.h b/llvm/include/llvm/MC/MCDXContainerStreamer.h new file mode 100644 index 000000000000..ef1a95f71778 --- /dev/null +++ b/llvm/include/llvm/MC/MCDXContainerStreamer.h @@ -0,0 +1,49 @@ +//===- MCDXContainerStreamer.h - MCDXContainerStreamer Interface ---*- C++ ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Overrides MCObjectStreamer to disable all unnecessary features with stubs. +// The DXContainer format isn't a fully featured object format. It doesn't +// support symbols, and initially it will not support instruction data since it +// is used as a bitcode container for DXIL. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCDXCONTAINERSTREAMER_H +#define LLVM_MC_MCDXCONTAINERSTREAMER_H + +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCObjectWriter.h" + +namespace llvm { +class MCAssembler; +class MCExpr; +class MCInst; +class raw_ostream; + +class MCDXContainerStreamer : public MCObjectStreamer { +public: + MCDXContainerStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, + std::unique_ptr<MCObjectWriter> OW, + std::unique_ptr<MCCodeEmitter> Emitter) + : MCObjectStreamer(Context, std::move(TAB), std::move(OW), + std::move(Emitter)) {} + + bool emitSymbolAttribute(MCSymbol *, MCSymbolAttr) override { return false; } + void emitCommonSymbol(MCSymbol *, uint64_t, unsigned) override {} + void emitZerofill(MCSection *, MCSymbol *Symbol = nullptr, uint64_t Size = 0, + unsigned ByteAlignment = 0, SMLoc Loc = SMLoc()) override {} + +private: + void emitInstToData(const MCInst &, const MCSubtargetInfo &) override; +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCDXCONTAINERSTREAMER_H diff --git a/llvm/include/llvm/MC/MCDXContainerWriter.h b/llvm/include/llvm/MC/MCDXContainerWriter.h new file mode 100644 index 000000000000..8ecb86c8a16f --- /dev/null +++ b/llvm/include/llvm/MC/MCDXContainerWriter.h @@ -0,0 +1,45 @@ +//===- llvm/MC/MCDXContainerWriter.h - DXContainer Writer -*- C++ -------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCDXCONTAINERWRITER_H +#define LLVM_MC_MCDXCONTAINERWRITER_H + +#include "llvm/ADT/Triple.h" +#include "llvm/MC/MCObjectWriter.h" + +namespace llvm { + +class raw_pwrite_stream; + +class MCDXContainerTargetWriter : public MCObjectTargetWriter { +protected: + MCDXContainerTargetWriter() {} + +public: + virtual ~MCDXContainerTargetWriter(); + + Triple::ObjectFormatType getFormat() const override { + return Triple::DXContainer; + } + static bool classof(const MCObjectTargetWriter *W) { + return W->getFormat() == Triple::DXContainer; + } +}; + +/// Construct a new DXContainer writer instance. +/// +/// \param MOTW - The target specific DXContainer writer subclass. +/// \param OS - The stream to write to. +/// \returns The constructed object writer. +std::unique_ptr<MCObjectWriter> +createDXContainerObjectWriter(std::unique_ptr<MCDXContainerTargetWriter> MOTW, + raw_pwrite_stream &OS); + +} // end namespace llvm + +#endif // LLVM_MC_MCDXCONTAINERWRITER_H diff --git a/llvm/include/llvm/MC/MCFixedLenDisassembler.h b/llvm/include/llvm/MC/MCDecoderOps.h index 1edf3899c130..c1956993fca2 100644 --- a/llvm/include/llvm/MC/MCFixedLenDisassembler.h +++ b/llvm/include/llvm/MC/MCDecoderOps.h @@ -1,14 +1,14 @@ -//===-- llvm/MC/MCFixedLenDisassembler.h - Decoder driver -------*- C++ -*-===// +//===------------ llvm/MC/MCDecoderOps.h - Decoder driver -------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// Fixed length disassembler decoder state machine driver. +// Disassembler decoder state machine driver. //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_MCFIXEDLENDISASSEMBLER_H -#define LLVM_MC_MCFIXEDLENDISASSEMBLER_H +#ifndef LLVM_MC_MCDECODEROPS_H +#define LLVM_MC_MCDECODEROPS_H namespace llvm { diff --git a/llvm/include/llvm/MC/MCDirectives.h b/llvm/include/llvm/MC/MCDirectives.h index 51e57ad37021..d6ab29febeeb 100644 --- a/llvm/include/llvm/MC/MCDirectives.h +++ b/llvm/include/llvm/MC/MCDirectives.h @@ -31,6 +31,7 @@ enum MCSymbolAttr { MCSA_LGlobal, ///< .lglobl (XCOFF) MCSA_Extern, ///< .extern (XCOFF) MCSA_Hidden, ///< .hidden (ELF) + MCSA_Exported, ///< .globl _foo, exported (XCOFF) MCSA_IndirectSymbol, ///< .indirect_symbol (MachO) MCSA_Internal, ///< .internal (ELF) MCSA_LazyReference, ///< .lazy_reference (MachO) diff --git a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h index 10037cd66ef1..de069ff95c2f 100644 --- a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h +++ b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h @@ -40,26 +40,35 @@ struct SymbolInfoTy { private: bool IsXCOFF; + bool HasType; public: SymbolInfoTy(uint64_t Addr, StringRef Name, Optional<XCOFF::StorageMappingClass> Smc, Optional<uint32_t> Idx, bool Label) - : Addr(Addr), Name(Name), XCOFFSymInfo(Smc, Idx, Label), IsXCOFF(true) {} - SymbolInfoTy(uint64_t Addr, StringRef Name, uint8_t Type) - : Addr(Addr), Name(Name), Type(Type), IsXCOFF(false) {} + : Addr(Addr), Name(Name), XCOFFSymInfo(Smc, Idx, Label), IsXCOFF(true), + HasType(false) {} + SymbolInfoTy(uint64_t Addr, StringRef Name, uint8_t Type, + bool IsXCOFF = false) + : Addr(Addr), Name(Name), Type(Type), IsXCOFF(IsXCOFF), HasType(true) {} bool isXCOFF() const { return IsXCOFF; } private: friend bool operator<(const SymbolInfoTy &P1, const SymbolInfoTy &P2) { - assert(P1.IsXCOFF == P2.IsXCOFF && - "P1.IsXCOFF should be equal to P2.IsXCOFF."); + assert((P1.IsXCOFF == P2.IsXCOFF && P1.HasType == P2.HasType) && + "The value of IsXCOFF and HasType in P1 and P2 should be the same " + "respectively."); + + if (P1.IsXCOFF && P1.HasType) + return std::tie(P1.Addr, P1.Type, P1.Name) < + std::tie(P2.Addr, P2.Type, P2.Name); + if (P1.IsXCOFF) return std::tie(P1.Addr, P1.XCOFFSymInfo, P1.Name) < std::tie(P2.Addr, P2.XCOFFSymInfo, P2.Name); return std::tie(P1.Addr, P1.Name, P1.Type) < - std::tie(P2.Addr, P2.Name, P2.Type); + std::tie(P2.Addr, P2.Name, P2.Type); } }; @@ -172,10 +181,9 @@ protected: public: // Helpers around MCSymbolizer - bool tryAddingSymbolicOperand(MCInst &Inst, - int64_t Value, - uint64_t Address, bool IsBranch, - uint64_t Offset, uint64_t InstSize) const; + bool tryAddingSymbolicOperand(MCInst &Inst, int64_t Value, uint64_t Address, + bool IsBranch, uint64_t Offset, uint64_t OpSize, + uint64_t InstSize) const; void tryAddingPcLoadReferenceComment(int64_t Value, uint64_t Address) const; diff --git a/llvm/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h b/llvm/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h index ffac5ee5cb1f..8af3bb2296ec 100644 --- a/llvm/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h +++ b/llvm/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h @@ -15,7 +15,7 @@ #ifndef LLVM_MC_MCDISASSEMBLER_MCEXTERNALSYMBOLIZER_H #define LLVM_MC_MCDISASSEMBLER_MCEXTERNALSYMBOLIZER_H -#include "llvm-c/Disassembler.h" +#include "llvm-c/DisassemblerTypes.h" #include "llvm/MC/MCDisassembler/MCSymbolizer.h" #include <memory> @@ -46,7 +46,8 @@ public: bool tryAddingSymbolicOperand(MCInst &MI, raw_ostream &CommentStream, int64_t Value, uint64_t Address, bool IsBranch, - uint64_t Offset, uint64_t InstSize) override; + uint64_t Offset, uint64_t OpSize, + uint64_t InstSize) override; void tryAddingPcLoadReferenceComment(raw_ostream &CommentStream, int64_t Value, uint64_t Address) override; diff --git a/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h b/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h index b966106007db..1efb63f1a142 100644 --- a/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h +++ b/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h @@ -17,9 +17,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/MC/MCDisassembler/MCRelocationInfo.h" -#include <algorithm> #include <cstdint> #include <memory> +#include <utility> namespace llvm { @@ -63,12 +63,13 @@ public: /// \param Address - Load address of the instruction. /// \param IsBranch - Is the instruction a branch? /// \param Offset - Byte offset of the operand inside the inst. + /// \param OpSize - Size of the operand in bytes. /// \param InstSize - Size of the instruction in bytes. /// \return Whether a symbolic operand was added. virtual bool tryAddingSymbolicOperand(MCInst &Inst, raw_ostream &cStream, int64_t Value, uint64_t Address, bool IsBranch, uint64_t Offset, - uint64_t InstSize) = 0; + uint64_t OpSize, uint64_t InstSize) = 0; /// Try to add a comment on the PC-relative load. /// For instance, in Mach-O, this is used to add annotations to instructions diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h index 7e72d56f3097..ce65b173b3d2 100644 --- a/llvm/include/llvm/MC/MCDwarf.h +++ b/llvm/include/llvm/MC/MCDwarf.h @@ -19,14 +19,12 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCSection.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/Error.h" #include "llvm/Support/MD5.h" #include <cassert> #include <cstdint> #include <string> -#include <tuple> #include <utility> #include <vector> @@ -36,6 +34,7 @@ template <typename T> class ArrayRef; class MCAsmBackend; class MCContext; class MCObjectStreamer; +class MCSection; class MCStreamer; class MCSymbol; class raw_ostream; @@ -63,6 +62,9 @@ public: /// Emit the .debug_line_str section if appropriate. void emitSection(MCStreamer *MCOS); + + /// Returns finalized section. + SmallString<0> getFinalizedData(); }; /// Instances of this class represent the name of the dwarf .file directive and @@ -294,8 +296,8 @@ public: RootFile.DirIndex = 0; RootFile.Checksum = Checksum; RootFile.Source = Source; - trackMD5Usage(Checksum.hasValue()); - HasSource = Source.hasValue(); + trackMD5Usage(Checksum.has_value()); + HasSource = Source.has_value(); } void resetFileTable() { @@ -686,6 +688,7 @@ struct MCDwarfFrameInfo { bool IsSimple = false; unsigned RAReg = static_cast<unsigned>(INT_MAX); bool IsBKeyFrame = false; + bool IsMTETaggedFrame = false; }; class MCDwarfFrameEmitter { diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h index 8f2b176862c8..eac807aad908 100644 --- a/llvm/include/llvm/MC/MCELFStreamer.h +++ b/llvm/include/llvm/MC/MCELFStreamer.h @@ -10,12 +10,19 @@ #define LLVM_MC_MCELFSTREAMER_H #include "llvm/ADT/SmallVector.h" -#include "llvm/BinaryFormat/ELF.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCObjectStreamer.h" namespace llvm { +class MCContext; +class MCDataFragment; +class MCFragment; +class MCObjectWriter; +class MCSection; +class MCSubtargetInfo; +class MCSymbol; +class MCSymbolRefExpr; class MCAsmBackend; class MCCodeEmitter; class MCExpr; diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h index 736fdd992063..b6329b131624 100644 --- a/llvm/include/llvm/MC/MCFragment.h +++ b/llvm/include/llvm/MC/MCFragment.h @@ -17,7 +17,6 @@ #include "llvm/MC/MCFixup.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/Alignment.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/SMLoc.h" #include <cstdint> #include <utility> @@ -294,7 +293,7 @@ public: class MCAlignFragment : public MCFragment { /// The alignment to ensure, in bytes. - unsigned Alignment; + Align Alignment; /// Flag to indicate that (optimal) NOPs should be emitted instead /// of using the provided value. The exact interpretation of this flag is @@ -315,12 +314,12 @@ class MCAlignFragment : public MCFragment { const MCSubtargetInfo *STI; public: - MCAlignFragment(unsigned Alignment, int64_t Value, unsigned ValueSize, + MCAlignFragment(Align Alignment, int64_t Value, unsigned ValueSize, unsigned MaxBytesToEmit, MCSection *Sec = nullptr) : MCFragment(FT_Align, false, Sec), Alignment(Alignment), EmitNops(false), Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {} - unsigned getAlignment() const { return Alignment; } + Align getAlignment() const { return Alignment; } int64_t getValue() const { return Value; } diff --git a/llvm/include/llvm/MC/MCInstrAnalysis.h b/llvm/include/llvm/MC/MCInstrAnalysis.h index 632a7d8f820e..a937f8203a0d 100644 --- a/llvm/include/llvm/MC/MCInstrAnalysis.h +++ b/llvm/include/llvm/MC/MCInstrAnalysis.h @@ -14,10 +14,13 @@ #ifndef LLVM_MC_MCINSTRANALYSIS_H #define LLVM_MC_MCINSTRANALYSIS_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCInstrInfo.h" #include <cstdint> +#include <vector> namespace llvm { diff --git a/llvm/include/llvm/MC/MCInstrDesc.h b/llvm/include/llvm/MC/MCInstrDesc.h index e8ffd29170e6..120c3482ce70 100644 --- a/llvm/include/llvm/MC/MCInstrDesc.h +++ b/llvm/include/llvm/MC/MCInstrDesc.h @@ -14,10 +14,11 @@ #ifndef LLVM_MC_MCINSTRDESC_H #define LLVM_MC_MCINSTRDESC_H -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/Support/DataTypes.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/MC/MCRegister.h" namespace llvm { +class MCRegisterInfo; class MCInst; @@ -148,6 +149,7 @@ enum Flag { Variadic, HasOptionalDef, Pseudo, + Meta, Return, EHScopeReturn, Call, @@ -263,6 +265,10 @@ public: /// correspond to a real machine instruction. bool isPseudo() const { return Flags & (1ULL << MCID::Pseudo); } + /// Return true if this is a meta instruction that doesn't + /// produce any output in the form of executable instructions. + bool isMetaInstruction() const { return Flags & (1ULL << MCID::Meta); } + /// Return true if the instruction is a return. bool isReturn() const { return Flags & (1ULL << MCID::Return); } diff --git a/llvm/include/llvm/MC/MCInstrInfo.h b/llvm/include/llvm/MC/MCInstrInfo.h index 598e24257e5d..84995b1e93fe 100644 --- a/llvm/include/llvm/MC/MCInstrInfo.h +++ b/llvm/include/llvm/MC/MCInstrInfo.h @@ -13,6 +13,7 @@ #ifndef LLVM_MC_MCINSTRINFO_H #define LLVM_MC_MCINSTRINFO_H +#include "llvm/ADT/StringRef.h" #include "llvm/MC/MCInstrDesc.h" #include <cassert> diff --git a/llvm/include/llvm/MC/MCLinkerOptimizationHint.h b/llvm/include/llvm/MC/MCLinkerOptimizationHint.h index 003491f32f75..b91fbc62aa75 100644 --- a/llvm/include/llvm/MC/MCLinkerOptimizationHint.h +++ b/llvm/include/llvm/MC/MCLinkerOptimizationHint.h @@ -19,7 +19,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" -#include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> @@ -28,6 +27,7 @@ namespace llvm { class MachObjectWriter; class MCAsmLayout; class MCSymbol; +class raw_ostream; /// Linker Optimization Hint Type. enum MCLOHType { diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h index f4f9c474cdcd..149373dd2b54 100644 --- a/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -264,6 +264,8 @@ public: bool IsPCRel) const override; uint64_t writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override; + + void writeAddrsigSection(MCAssembler &Asm); }; /// Construct a new Mach-O writer instance. diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h index 3c1d10c4e62f..ebc9b95d6d4e 100644 --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -13,13 +13,13 @@ #ifndef LLVM_MC_MCOBJECTFILEINFO_H #define LLVM_MC_MCOBJECTFILEINFO_H -#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/Swift.h" -#include "llvm/MC/MCSymbol.h" -#include "llvm/Support/CodeGen.h" #include "llvm/Support/VersionTuple.h" +#include <array> + namespace llvm { class MCContext; class MCSection; @@ -213,6 +213,7 @@ protected: MCSection *LazySymbolPointerSection = nullptr; MCSection *NonLazySymbolPointerSection = nullptr; MCSection *ThreadLocalPointerSection = nullptr; + MCSection *AddrSigSection = nullptr; /// COFF specific sections. MCSection *DrectveSection = nullptr; @@ -224,6 +225,9 @@ protected: MCSection *GIATsSection = nullptr; MCSection *GLJMPSection = nullptr; + // GOFF specific sections. + MCSection *PPA1Section = nullptr; + // XCOFF specific sections MCSection *TOCBaseSection = nullptr; MCSection *ReadOnly8Section = nullptr; @@ -410,6 +414,7 @@ public: MCSection *getThreadLocalPointerSection() const { return ThreadLocalPointerSection; } + MCSection *getAddrSigSection() const { return AddrSigSection; } // COFF specific sections. MCSection *getDrectveSection() const { return DrectveSection; } @@ -421,6 +426,9 @@ public: MCSection *getGIATsSection() const { return GIATsSection; } MCSection *getGLJMPSection() const { return GLJMPSection; } + // GOFF specific sections. + MCSection *getPPA1Section() const { return PPA1Section; } + // XCOFF specific sections MCSection *getTOCBaseSection() const { return TOCBaseSection; } @@ -448,8 +456,10 @@ private: void initELFMCObjectFileInfo(const Triple &T, bool Large); void initGOFFMCObjectFileInfo(const Triple &T); void initCOFFMCObjectFileInfo(const Triple &T); + void initSPIRVMCObjectFileInfo(const Triple &T); void initWasmMCObjectFileInfo(const Triple &T); void initXCOFFMCObjectFileInfo(const Triple &T); + void initDXContainerObjectFileInfo(const Triple &T); MCSection *getDwarfComdatSection(const char *Name, uint64_t Hash) const; public: diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index 183fd79fb9fc..6536c81d4aac 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -11,11 +11,17 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/MC/MCAssembler.h" +#include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCFragment.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" namespace llvm { +class MCContext; +class MCInst; +class MCObjectWriter; +class MCSymbol; +struct MCDwarfFrameInfo; class MCAssembler; class MCCodeEmitter; class MCSubtargetInfo; diff --git a/llvm/include/llvm/MC/MCObjectWriter.h b/llvm/include/llvm/MC/MCObjectWriter.h index d2a2f1a13ff5..a8e24a0c56ba 100644 --- a/llvm/include/llvm/MC/MCObjectWriter.h +++ b/llvm/include/llvm/MC/MCObjectWriter.h @@ -10,6 +10,7 @@ #define LLVM_MC_MCOBJECTWRITER_H #include "llvm/ADT/Triple.h" +#include "llvm/MC/MCSymbol.h" #include <cstdint> namespace llvm { @@ -32,6 +33,9 @@ class MCValue; /// should be emitted as part of writeObject(). class MCObjectWriter { protected: + std::vector<const MCSymbol *> AddrsigSyms; + bool EmitAddrsigSection = false; + MCObjectWriter() = default; public: @@ -91,11 +95,15 @@ public: /// Tell the object writer to emit an address-significance table during /// writeObject(). If this function is not called, all symbols are treated as /// address-significant. - virtual void emitAddrsigSection() {} + void emitAddrsigSection() { EmitAddrsigSection = true; } + + bool getEmitAddrsigSection() { return EmitAddrsigSection; } /// Record the given symbol in the address-significance table to be written /// diring writeObject(). - virtual void addAddrsigSymbol(const MCSymbol *Sym) {} + void addAddrsigSymbol(const MCSymbol *Sym) { AddrsigSyms.push_back(Sym); } + + std::vector<const MCSymbol *> &getAddrsigSyms() { return AddrsigSyms; } /// Write the object file and returns the number of bytes written. /// diff --git a/llvm/include/llvm/MC/MCParser/MCAsmLexer.h b/llvm/include/llvm/MC/MCParser/MCAsmLexer.h index 06796979b4fc..850a9cffe73a 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmLexer.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmLexer.h @@ -12,10 +12,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCAsmMacro.h" -#include <algorithm> #include <cassert> #include <cstddef> -#include <cstdint> #include <string> namespace llvm { diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h index 29386ffc45ac..4a1291856a20 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -10,20 +10,20 @@ #define LLVM_MC_MCPARSER_MCASMPARSER_H #include "llvm/ADT/None.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" -#include "llvm/MC/MCParser/MCAsmLexer.h" +#include "llvm/MC/MCAsmMacro.h" #include "llvm/Support/SMLoc.h" #include <cstdint> -#include <ctime> #include <string> #include <utility> namespace llvm { +class MCAsmLexer; class MCAsmInfo; class MCAsmParserExtension; class MCContext; diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h index fc10e33bcf6b..cbabc2c9d69d 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h @@ -9,9 +9,8 @@ #ifndef LLVM_MC_MCPARSER_MCASMPARSEREXTENSION_H #define LLVM_MC_MCPARSER_MCASMPARSEREXTENSION_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCParser/MCAsmLexer.h" #include "llvm/MC/MCParser/MCAsmParser.h" #include "llvm/Support/SMLoc.h" diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index faf0a4474c8a..22f66a011ece 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -10,7 +10,6 @@ #define LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H #include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/SMLoc.h" #include <string> @@ -63,6 +62,13 @@ public: /// isMem - Is this a memory operand? virtual bool isMem() const = 0; + /// isMemUseUpRegs - Is memory operand use up regs, for example, intel MS + /// inline asm may use ARR[baseReg + IndexReg + ...] which may use up regs + /// in [...] expr, so ARR[baseReg + IndexReg + ...] can not use extra reg + /// for ARR. For example, calculating ARR address to a reg or use another + /// base reg in PIC model. + virtual bool isMemUseUpRegs() const { return false; } + /// getStartLoc - Get the location of the first token of this operand. virtual SMLoc getStartLoc() const = 0; /// getEndLoc - Get the location of the last token of this operand. @@ -77,10 +83,6 @@ public: /// assembly. virtual bool isOffsetOfLocal() const { return false; } - /// isMemPlaceholder - Do we need to ignore the constraint, rather than emit - /// code? Only valid when parsing MS-style inline assembly. - virtual bool isMemPlaceholder(const MCInstrDesc &Desc) const { return false; } - /// getOffsetOfLoc - Get the location of the offset operator. virtual SMLoc getOffsetOfLoc() const { return SMLoc(); } diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 908ee30e4060..1d380c6a00b7 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -11,10 +11,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCExpr.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCParser/MCAsmLexer.h" -#include "llvm/MC/MCParser/MCParsedAsmOperand.h" #include "llvm/MC/MCParser/MCAsmParserExtension.h" +#include "llvm/MC/MCParser/MCParsedAsmOperand.h" #include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Support/SMLoc.h" @@ -23,9 +21,12 @@ namespace llvm { +class MCContext; class MCInst; +class MCInstrInfo; class MCStreamer; class MCSubtargetInfo; +class MCSymbol; template <typename T> class SmallVectorImpl; using OperandVector = SmallVectorImpl<std::unique_ptr<MCParsedAsmOperand>>; @@ -100,10 +101,14 @@ struct AsmRewrite { int64_t Val; StringRef Label; IntelExpr IntelExp; + bool IntelExpRestricted; public: - AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, int64_t val = 0) - : Kind(kind), Loc(loc), Len(len), Done(false), Val(val) {} + AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, int64_t val = 0, + bool Restricted = false) + : Kind(kind), Loc(loc), Len(len), Done(false), Val(val) { + IntelExpRestricted = Restricted; + } AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len, StringRef label) : AsmRewrite(kind, loc, len) { Label = label; } AsmRewrite(SMLoc loc, unsigned len, IntelExpr exp) diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h index 9ff68f4236ca..d10d6015cd3c 100644 --- a/llvm/include/llvm/MC/MCPseudoProbe.h +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -55,6 +55,7 @@ #include <tuple> #include <type_traits> #include <unordered_map> +#include <unordered_set> #include <vector> namespace llvm { @@ -82,10 +83,9 @@ struct MCPseudoProbeFuncDesc { void print(raw_ostream &OS); }; -class MCPseudoProbe; class MCDecodedPseudoProbe; -// An inline frame has the form <Guid, ProbeID> +// An inline frame has the form <CalleeGuid, ProbeID> using InlineSite = std::tuple<uint64_t, uint32_t>; using MCPseudoProbeInlineStack = SmallVector<InlineSite, 8>; // GUID to PseudoProbeFuncDesc map @@ -95,7 +95,6 @@ using GUIDProbeFunctionMap = using AddressProbesMap = std::unordered_map<uint64_t, std::list<MCDecodedPseudoProbe>>; -class MCPseudoProbeInlineTree; class MCDecodedPseudoProbeInlineTree; class MCPseudoProbeBase { @@ -272,7 +271,7 @@ public: MCDecodedPseudoProbeInlineTree(const InlineSite &Site) : ISite(Site){}; // Return false if it's a dummy inline site - bool hasInlineSite() const { return std::get<0>(ISite) != 0; } + bool hasInlineSite() const { return !isRoot() && !Parent->isRoot(); } }; /// Instances of this class represent the pseudo probes inserted into a compile @@ -355,6 +354,15 @@ public: // Decode pseudo_probe section to build address to probes map. bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size); + // Decode pseudo_probe section to build address to probes map for specifed + // functions only. + bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size, + std::unordered_set<uint64_t> &GuildFilter); + + bool buildAddress2ProbeMap(MCDecodedPseudoProbeInlineTree *Cur, + uint64_t &LastAddr, + std::unordered_set<uint64_t> &GuildFilter); + // Print pseudo_probe_desc section info void printGUID2FuncDescMap(raw_ostream &OS); diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index 65436dc74c3e..7165a2982d1b 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -580,6 +580,9 @@ public: bool isSuperOrSubRegisterEq(MCRegister RegA, MCRegister RegB) const { return isSubRegisterEq(RegA, RegB) || isSuperRegister(RegA, RegB); } + + /// Returns true if the two registers are equal or alias each other. + bool regsOverlap(MCRegister RegA, MCRegister RegB) const; }; //===----------------------------------------------------------------------===// @@ -698,6 +701,11 @@ public: // unit, we can allow a 0 differential here. advance(); } + + MCRegUnitIterator &operator++() { + MCRegisterInfo::DiffListIterator::operator++(); + return *this; + } }; /// MCRegUnitMaskIterator enumerates a list of register units and their diff --git a/llvm/include/llvm/MC/MCSPIRVObjectWriter.h b/llvm/include/llvm/MC/MCSPIRVObjectWriter.h new file mode 100644 index 000000000000..a8baf96b8384 --- /dev/null +++ b/llvm/include/llvm/MC/MCSPIRVObjectWriter.h @@ -0,0 +1,40 @@ +//===-- llvm/MC/MCSPIRVObjectWriter.h - SPIR-V Object Writer -----*- C++ *-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCSPIRVOBJECTWRITER_H +#define LLVM_MC_MCSPIRVOBJECTWRITER_H + +#include "llvm/MC/MCObjectWriter.h" +#include "llvm/Support/raw_ostream.h" +#include <memory> + +namespace llvm { + +class MCSPIRVObjectTargetWriter : public MCObjectTargetWriter { +protected: + explicit MCSPIRVObjectTargetWriter() {} + +public: + Triple::ObjectFormatType getFormat() const override { return Triple::SPIRV; } + static bool classof(const MCObjectTargetWriter *W) { + return W->getFormat() == Triple::SPIRV; + } +}; + +/// Construct a new SPIR-V writer instance. +/// +/// \param MOTW - The target specific SPIR-V writer subclass. +/// \param OS - The stream to write to. +/// \returns The constructed object writer. +std::unique_ptr<MCObjectWriter> +createSPIRVObjectWriter(std::unique_ptr<MCSPIRVObjectTargetWriter> MOTW, + raw_pwrite_stream &OS); + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/MC/MCSPIRVStreamer.h b/llvm/include/llvm/MC/MCSPIRVStreamer.h new file mode 100644 index 000000000000..7366e0a9d82c --- /dev/null +++ b/llvm/include/llvm/MC/MCSPIRVStreamer.h @@ -0,0 +1,50 @@ +//===- MCSPIRVStreamer.h - MCStreamer SPIR-V Object File Interface -*- C++ ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Overrides MCObjectStreamer to disable all unnecessary features with stubs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCSPIRVSTREAMER_H +#define LLVM_MC_MCSPIRVSTREAMER_H + +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCObjectWriter.h" + +namespace llvm { +class MCAssembler; +class MCExpr; +class MCInst; +class raw_ostream; + +class MCSPIRVStreamer : public MCObjectStreamer { +public: + MCSPIRVStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, + std::unique_ptr<MCObjectWriter> OW, + std::unique_ptr<MCCodeEmitter> Emitter) + : MCObjectStreamer(Context, std::move(TAB), std::move(OW), + std::move(Emitter)) {} + + bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override { + return false; + } + void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment) override {} + void emitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr, + uint64_t Size = 0, unsigned ByteAlignment = 0, + SMLoc Loc = SMLoc()) override {} + +private: + void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override; +}; + +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 4335092f0920..2f7e17123c19 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -46,7 +46,9 @@ public: SV_GOFF, SV_MachO, SV_Wasm, - SV_XCOFF + SV_XCOFF, + SV_SPIRV, + SV_DXContainer, }; /// Express the state of bundle locked groups while emitting code. @@ -184,13 +186,13 @@ public: void dump() const; - virtual void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + virtual void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const = 0; /// Return true if a .align directive should use "optimized nops" to fill /// instead of 0s. - virtual bool UseCodeAlign() const = 0; + virtual bool useCodeAlign() const = 0; /// Check whether this section is "virtual", that is has no actual object /// file contents. diff --git a/llvm/include/llvm/MC/MCSectionCOFF.h b/llvm/include/llvm/MC/MCSectionCOFF.h index 3ece6eb904bc..373863e21ff0 100644 --- a/llvm/include/llvm/MC/MCSectionCOFF.h +++ b/llvm/include/llvm/MC/MCSectionCOFF.h @@ -61,7 +61,7 @@ private: public: /// Decides whether a '.section' directive should be printed before the /// section name - bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; + bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; unsigned getCharacteristics() const { return Characteristics; } MCSymbol *getCOMDATSymbol() const { return COMDATSymbol; } @@ -69,10 +69,10 @@ public: void setSelection(int Selection) const; - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; + bool useCodeAlign() const override; bool isVirtualSection() const override; StringRef getVirtualSectionKind() const override; diff --git a/llvm/include/llvm/MC/MCSectionDXContainer.h b/llvm/include/llvm/MC/MCSectionDXContainer.h new file mode 100644 index 000000000000..014684a93529 --- /dev/null +++ b/llvm/include/llvm/MC/MCSectionDXContainer.h @@ -0,0 +1,38 @@ +//===- MCSectionDXContainer.h - DXContainer MC Sections ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the MCSectionDXContainer class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCSECTIONDXCONTAINER_H +#define LLVM_MC_MCSECTIONDXCONTAINER_H + +#include "llvm/MC/MCSection.h" +#include "llvm/MC/SectionKind.h" + +namespace llvm { + +class MCSymbol; + +class MCSectionDXContainer final : public MCSection { + friend class MCContext; + + MCSectionDXContainer(StringRef Name, SectionKind K, MCSymbol *Begin) + : MCSection(SV_DXContainer, Name, K, Begin) {} + +public: + void printSwitchToSection(const MCAsmInfo &, const Triple &, raw_ostream &, + const MCExpr *) const override; + bool useCodeAlign() const override { return false; } + bool isVirtualSection() const override { return false; } +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCSECTIONDXCONTAINER_H diff --git a/llvm/include/llvm/MC/MCSectionELF.h b/llvm/include/llvm/MC/MCSectionELF.h index 8b17df25a158..3b5239394493 100644 --- a/llvm/include/llvm/MC/MCSectionELF.h +++ b/llvm/include/llvm/MC/MCSectionELF.h @@ -21,8 +21,6 @@ namespace llvm { -class MCSymbol; - /// This represents a section on linux, lots of unix variants and some bare /// metal systems. class MCSectionELF final : public MCSection { @@ -69,7 +67,7 @@ private: public: /// Decides whether a '.section' directive should be printed before the /// section name - bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; + bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; unsigned getType() const { return Type; } unsigned getFlags() const { return Flags; } @@ -78,10 +76,10 @@ public: const MCSymbolELF *getGroup() const { return Group.getPointer(); } bool isComdat() const { return Group.getInt(); } - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; + bool useCodeAlign() const override; bool isVirtualSection() const override; StringRef getVirtualSectionKind() const override; diff --git a/llvm/include/llvm/MC/MCSectionGOFF.h b/llvm/include/llvm/MC/MCSectionGOFF.h index 4ba7f79f9696..d866329461ce 100644 --- a/llvm/include/llvm/MC/MCSectionGOFF.h +++ b/llvm/include/llvm/MC/MCSectionGOFF.h @@ -15,6 +15,7 @@ #ifndef LLVM_MC_MCSECTIONGOFF_H #define LLVM_MC_MCSECTIONGOFF_H +#include "llvm/BinaryFormat/GOFF.h" #include "llvm/MC/MCSection.h" #include "llvm/Support/raw_ostream.h" @@ -24,21 +25,27 @@ class MCExpr; class MCSectionGOFF final : public MCSection { private: + MCSection *Parent; + const MCExpr *SubsectionId; + friend class MCContext; - MCSectionGOFF(StringRef Name, SectionKind K) - : MCSection(SV_GOFF, Name, K, nullptr) {} + MCSectionGOFF(StringRef Name, SectionKind K, MCSection *P, const MCExpr *Sub) + : MCSection(SV_GOFF, Name, K, nullptr), Parent(P), SubsectionId(Sub) {} public: - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override { OS << "\t.section\t\"" << getName() << "\"\n"; } - bool UseCodeAlign() const override { return false; } + bool useCodeAlign() const override { return false; } bool isVirtualSection() const override { return false; } + MCSection *getParent() const { return Parent; } + const MCExpr *getSubsectionId() const { return SubsectionId; } + static bool classof(const MCSection *S) { return S->getVariant() == SV_GOFF; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionMachO.h b/llvm/include/llvm/MC/MCSectionMachO.h index bf8940524e5a..fdf1773d4002 100644 --- a/llvm/include/llvm/MC/MCSectionMachO.h +++ b/llvm/include/llvm/MC/MCSectionMachO.h @@ -68,10 +68,10 @@ public: bool &TAAParsed, // Out. unsigned &StubSize); // Out. - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; + bool useCodeAlign() const override; bool isVirtualSection() const override; static bool classof(const MCSection *S) { diff --git a/llvm/include/llvm/MC/MCSectionSPIRV.h b/llvm/include/llvm/MC/MCSectionSPIRV.h new file mode 100644 index 000000000000..6534599d2091 --- /dev/null +++ b/llvm/include/llvm/MC/MCSectionSPIRV.h @@ -0,0 +1,41 @@ +//===- MCSectionSPIRV.h - SPIR-V Machine Code Sections ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the MCSectionSPIRV class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCSECTIONSPIRV_H +#define LLVM_MC_MCSECTIONSPIRV_H + +#include "llvm/MC/MCSection.h" +#include "llvm/MC/SectionKind.h" + +namespace llvm { + +class MCSymbol; + +class MCSectionSPIRV final : public MCSection { + friend class MCContext; + + MCSectionSPIRV(SectionKind K, MCSymbol *Begin) + : MCSection(SV_SPIRV, "", K, Begin) {} + // TODO: Add StringRef Name to MCSectionSPIRV. + +public: + ~MCSectionSPIRV() = default; + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + raw_ostream &OS, + const MCExpr *Subsection) const override {} + bool useCodeAlign() const override { return false; } + bool isVirtualSection() const override { return false; } +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCSECTIONSPIRV_H diff --git a/llvm/include/llvm/MC/MCSectionWasm.h b/llvm/include/llvm/MC/MCSectionWasm.h index f34dd6b3507c..579f92a75056 100644 --- a/llvm/include/llvm/MC/MCSectionWasm.h +++ b/llvm/include/llvm/MC/MCSectionWasm.h @@ -58,10 +58,10 @@ public: const MCSymbolWasm *getGroup() const { return Group; } unsigned getSegmentFlags() const { return SegmentFlags; } - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; + bool useCodeAlign() const override; bool isVirtualSection() const override; bool isWasmData() const { diff --git a/llvm/include/llvm/MC/MCSectionXCOFF.h b/llvm/include/llvm/MC/MCSectionXCOFF.h index 1dafdd3ac500..95332647c9be 100644 --- a/llvm/include/llvm/MC/MCSectionXCOFF.h +++ b/llvm/include/llvm/MC/MCSectionXCOFF.h @@ -38,6 +38,7 @@ class MCSectionXCOFF final : public MCSection { Optional<XCOFF::DwarfSectionSubtypeFlags> DwarfSubtypeFlags; bool MultiSymbolsAllowed; static constexpr unsigned DefaultAlignVal = 4; + static constexpr unsigned DefaultTextAlignVal = 32; MCSectionXCOFF(StringRef Name, XCOFF::StorageMappingClass SMC, XCOFF::SymbolType ST, SectionKind K, MCSymbolXCOFF *QualName, @@ -57,9 +58,14 @@ class MCSectionXCOFF final : public MCSection { QualName->setRepresentedCsect(this); QualName->setStorageClass(XCOFF::C_HIDEXT); - // A csect is 4 byte aligned by default, except for undefined symbol csects. - if (ST != XCOFF::XTY_ER) - setAlignment(Align(DefaultAlignVal)); + if (ST != XCOFF::XTY_ER) { + // For a csect for program code, set the alignment to 32 bytes by default. + // For other csects, set the alignment to 4 bytes by default. + if (SMC == XCOFF::XMC_PR) + setAlignment(Align(DefaultTextAlignVal)); + else + setAlignment(Align(DefaultAlignVal)); + } } MCSectionXCOFF(StringRef Name, SectionKind K, MCSymbolXCOFF *QualName, @@ -74,9 +80,8 @@ class MCSectionXCOFF final : public MCSection { // FIXME: use a more meaningful name for non csect sections. QualName->setRepresentedCsect(this); - // Set default alignment 4 for all non csect sections for now. - // FIXME: set different alignments according to section types. - setAlignment(Align(DefaultAlignVal)); + // Use default text alignment as the alignment for DWARF sections. + setAlignment(Align(DefaultTextAlignVal)); } void printCsectDirective(raw_ostream &OS) const; @@ -95,24 +100,28 @@ public: XCOFF::StorageClass getStorageClass() const { return QualName->getStorageClass(); } + XCOFF::VisibilityType getVisibilityType() const { + return QualName->getVisibilityType(); + } XCOFF::SymbolType getCSectType() const { assert(isCsect() && "Only csect section has symbol type property!"); return CsectProp->Type; } MCSymbolXCOFF *getQualNameSymbol() const { return QualName; } - void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; + bool useCodeAlign() const override; bool isVirtualSection() const override; StringRef getSymbolTableName() const { return SymbolTableName; } bool isMultiSymbolsAllowed() const { return MultiSymbolsAllowed; } - bool isCsect() const { return CsectProp.hasValue(); } - bool isDwarfSect() const { return DwarfSubtypeFlags.hasValue(); } + bool isCsect() const { return CsectProp.has_value(); } + bool isDwarfSect() const { return DwarfSubtypeFlags.has_value(); } Optional<XCOFF::DwarfSectionSubtypeFlags> getDwarfSubtypeFlags() const { return DwarfSubtypeFlags; } + Optional<XCOFF::CsectProperties> getCsectProp() const { return CsectProp; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 3d6c512bfe73..e71014b8cccf 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -13,22 +13,20 @@ #ifndef LLVM_MC_MCSTREAMER_H #define LLVM_MC_MCSTREAMER_H -#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCDirectives.h" +#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCPseudoProbe.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" +#include "llvm/Support/ARMTargetParser.h" #include "llvm/Support/Error.h" #include "llvm/Support/MD5.h" #include "llvm/Support/SMLoc.h" -#include "llvm/Support/ARMTargetParser.h" -#include "llvm/Support/TargetParser.h" #include "llvm/Support/VersionTuple.h" #include <cassert> #include <cstdint> @@ -39,20 +37,24 @@ namespace llvm { +class APInt; class AssemblerConstantPools; class MCAsmBackend; +class MCAssembler; class MCContext; -struct MCDwarfFrameInfo; class MCExpr; +class MCFragment; class MCInst; class MCInstPrinter; class MCRegister; class MCSection; class MCStreamer; -class MCSymbolRefExpr; class MCSubtargetInfo; -class raw_ostream; +class MCSymbol; +class MCSymbolRefExpr; +class Triple; class Twine; +class raw_ostream; namespace codeview { struct DefRangeRegisterRelHeader; @@ -111,7 +113,7 @@ public: /// Update streamer for a new active section. /// - /// This is called by PopSection and SwitchSection, if the current + /// This is called by popSection and switchSection, if the current /// section changes. virtual void changeSection(const MCSection *CurSection, MCSection *Section, const MCExpr *SubSection, raw_ostream &OS); @@ -163,12 +165,23 @@ public: virtual void finishAttributeSection(); virtual void emitInst(uint32_t Inst, char Suffix = '\0'); - virtual void AnnotateTLSDescriptorSequence(const MCSymbolRefExpr *SRE); + virtual void annotateTLSDescriptorSequence(const MCSymbolRefExpr *SRE); virtual void emitThumbSet(MCSymbol *Symbol, const MCExpr *Value); void emitConstantPools() override; + virtual void emitARMWinCFIAllocStack(unsigned Size, bool Wide); + virtual void emitARMWinCFISaveRegMask(unsigned Mask, bool Wide); + virtual void emitARMWinCFISaveSP(unsigned Reg); + virtual void emitARMWinCFISaveFRegs(unsigned First, unsigned Last); + virtual void emitARMWinCFISaveLR(unsigned Offset); + virtual void emitARMWinCFIPrologEnd(bool Fragment); + virtual void emitARMWinCFINop(bool Wide); + virtual void emitARMWinCFIEpilogStart(unsigned Condition); + virtual void emitARMWinCFIEpilogEnd(); + virtual void emitARMWinCFICustom(unsigned Opcode); + /// Reset any state between object emissions, i.e. the equivalent of /// MCStreamer's reset method. virtual void reset(); @@ -215,7 +228,7 @@ class MCStreamer { DenseMap<const MCSymbol *, unsigned> SymbolOrdering; /// This is stack of current and previous section values saved by - /// PushSection. + /// pushSection. SmallVector<std::pair<MCSectionSubPair, MCSectionSubPair>, 4> SectionStack; /// Pointer to the parser's SMLoc if available. This is used to provide @@ -247,9 +260,9 @@ protected: return CurrentWinFrameInfo; } - virtual void EmitWindowsUnwindTables(WinEH::FrameInfo *Frame); + virtual void emitWindowsUnwindTables(WinEH::FrameInfo *Frame); - virtual void EmitWindowsUnwindTables(); + virtual void emitWindowsUnwindTables(); virtual void emitRawTextImpl(StringRef String); @@ -344,7 +357,7 @@ public: /// Return a raw_ostream that comments can be written to. Unlike /// AddComment, you are required to terminate comments with \n if you use this /// method. - virtual raw_ostream &GetCommentOS(); + virtual raw_ostream &getCommentOS(); /// Print T and prefix it with the comment string (normally #) and /// optionally a tab. This prints the comment immediately, not at the end of @@ -359,8 +372,8 @@ public: /// Emit added explicit comments. virtual void emitExplicitComments(); - /// AddBlankLine - Emit a blank line to a .s file to pretty it up. - virtual void AddBlankLine() {} + /// Emit a blank line to a .s file to pretty it up. + virtual void addBlankLine() {} /// @} @@ -384,18 +397,18 @@ public: /// Returns an index to represent the order a symbol was emitted in. /// (zero if we did not emit that symbol) - unsigned GetSymbolOrder(const MCSymbol *Sym) const { + unsigned getSymbolOrder(const MCSymbol *Sym) const { return SymbolOrdering.lookup(Sym); } /// Update streamer for a new active section. /// - /// This is called by PopSection and SwitchSection, if the current + /// This is called by popSection and switchSection, if the current /// section changes. virtual void changeSection(MCSection *, const MCExpr *); /// Save the current and previous section on the section stack. - void PushSection() { + void pushSection() { SectionStack.push_back( std::make_pair(getCurrentSection(), getPreviousSection())); } @@ -404,7 +417,7 @@ public: /// Calls changeSection as needed. /// /// Returns false if the stack was empty. - bool PopSection() { + bool popSection() { if (SectionStack.size() <= 1) return false; auto I = SectionStack.end(); @@ -419,11 +432,11 @@ public: return true; } - bool SubSection(const MCExpr *Subsection) { + bool subSection(const MCExpr *Subsection) { if (SectionStack.empty()) return false; - SwitchSection(SectionStack.back().first.first, Subsection); + switchSection(SectionStack.back().first.first, Subsection); return true; } @@ -431,13 +444,13 @@ public: /// is required to update CurSection. /// /// This corresponds to assembler directives like .section, .text, etc. - virtual void SwitchSection(MCSection *Section, + virtual void switchSection(MCSection *Section, const MCExpr *Subsection = nullptr); /// Set the current section where code is being emitted to \p Section. /// This is required to update CurSection. This version does not call /// changeSection. - void SwitchSectionNoChange(MCSection *Section, + void switchSectionNoChange(MCSection *Section, const MCExpr *Subsection = nullptr) { assert(Section && "Cannot switch to a null section!"); MCSectionSubPair curSection = SectionStack.back().first; @@ -455,7 +468,7 @@ public: /// /// Each emitted symbol will be tracked in the ordering table, /// so we can sort on them later. - void AssignFragment(MCSymbol *Symbol, MCFragment *Fragment); + void assignFragment(MCSymbol *Symbol, MCFragment *Fragment); /// Returns the mnemonic for \p MI, if the streamer has access to a /// instruction printer and returns an empty string otherwise. @@ -550,40 +563,40 @@ public: /// Start emitting COFF symbol definition /// /// \param Symbol - The symbol to have its External & Type fields set. - virtual void BeginCOFFSymbolDef(const MCSymbol *Symbol); + virtual void beginCOFFSymbolDef(const MCSymbol *Symbol); /// Emit the storage class of the symbol. /// /// \param StorageClass - The storage class the symbol should have. - virtual void EmitCOFFSymbolStorageClass(int StorageClass); + virtual void emitCOFFSymbolStorageClass(int StorageClass); /// Emit the type of the symbol. /// /// \param Type - A COFF type identifier (see COFF::SymbolType in X86COFF.h) - virtual void EmitCOFFSymbolType(int Type); + virtual void emitCOFFSymbolType(int Type); /// Marks the end of the symbol definition. - virtual void EndCOFFSymbolDef(); + virtual void endCOFFSymbolDef(); - virtual void EmitCOFFSafeSEH(MCSymbol const *Symbol); + virtual void emitCOFFSafeSEH(MCSymbol const *Symbol); /// Emits the symbol table index of a Symbol into the current section. - virtual void EmitCOFFSymbolIndex(MCSymbol const *Symbol); + virtual void emitCOFFSymbolIndex(MCSymbol const *Symbol); /// Emits a COFF section index. /// /// \param Symbol - Symbol the section number relocation should point to. - virtual void EmitCOFFSectionIndex(MCSymbol const *Symbol); + virtual void emitCOFFSectionIndex(MCSymbol const *Symbol); /// Emits a COFF section relative relocation. /// /// \param Symbol - Symbol the section relative relocation should point to. - virtual void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset); + virtual void emitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset); /// Emits a COFF image relative relocation. /// /// \param Symbol - Symbol the image relative relocation should point to. - virtual void EmitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset); + virtual void emitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset); /// Emits an lcomm directive with XCOFF csect information. /// @@ -615,6 +628,12 @@ public: /// changed at the end of assembly. virtual void emitXCOFFRenameDirective(const MCSymbol *Name, StringRef Rename); + /// Emit a XCOFF .ref directive which creates R_REF type entry in the + /// relocation table for one or more symbols. + /// + /// \param Sym - The symbol on the .ref directive. + virtual void emitXCOFFRefDirective(StringRef Sym); + /// Emit an ELF .size directive. /// /// This corresponds to an assembler statement such as: @@ -907,6 +926,7 @@ public: unsigned CUID = 0); virtual void emitCFIBKeyFrame(); + virtual void emitCFIMTETaggedFrame(); /// This implements the DWARF2 '.loc fileno lineno ...' assembler /// directive. @@ -918,16 +938,16 @@ public: /// Associate a filename with a specified logical file number, and also /// specify that file's checksum information. This implements the '.cv_file 4 /// "foo.c"' assembler directive. Returns true on success. - virtual bool EmitCVFileDirective(unsigned FileNo, StringRef Filename, + virtual bool emitCVFileDirective(unsigned FileNo, StringRef Filename, ArrayRef<uint8_t> Checksum, unsigned ChecksumKind); /// Introduces a function id for use with .cv_loc. - virtual bool EmitCVFuncIdDirective(unsigned FunctionId); + virtual bool emitCVFuncIdDirective(unsigned FunctionId); /// Introduces an inline call site id for use with .cv_loc. Includes /// extra information for inline line table generation. - virtual bool EmitCVInlineSiteIdDirective(unsigned FunctionId, unsigned IAFunc, + virtual bool emitCVInlineSiteIdDirective(unsigned FunctionId, unsigned IAFunc, unsigned IAFile, unsigned IALine, unsigned IACol, SMLoc Loc); @@ -983,7 +1003,7 @@ public: virtual void emitCVFileChecksumOffsetDirective(unsigned FileNo) {} /// This implements the CodeView '.cv_fpo_data' assembler directive. - virtual void EmitCVFPOData(const MCSymbol *ProcSym, SMLoc Loc = {}) {} + virtual void emitCVFPOData(const MCSymbol *ProcSym, SMLoc Loc = {}) {} /// Emit the absolute difference between two symbols. /// @@ -1022,28 +1042,28 @@ public: virtual void emitCFIWindowSave(); virtual void emitCFINegateRAState(); - virtual void EmitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc = SMLoc()); - virtual void EmitWinCFIEndProc(SMLoc Loc = SMLoc()); + virtual void emitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc = SMLoc()); + virtual void emitWinCFIEndProc(SMLoc Loc = SMLoc()); /// This is used on platforms, such as Windows on ARM64, that require function /// or funclet sizes to be emitted in .xdata before the End marker is emitted /// for the frame. We cannot use the End marker, as it is not set at the /// point of emitting .xdata, in order to indicate that the frame is active. - virtual void EmitWinCFIFuncletOrFuncEnd(SMLoc Loc = SMLoc()); - virtual void EmitWinCFIStartChained(SMLoc Loc = SMLoc()); - virtual void EmitWinCFIEndChained(SMLoc Loc = SMLoc()); - virtual void EmitWinCFIPushReg(MCRegister Register, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISetFrame(MCRegister Register, unsigned Offset, + virtual void emitWinCFIFuncletOrFuncEnd(SMLoc Loc = SMLoc()); + virtual void emitWinCFIStartChained(SMLoc Loc = SMLoc()); + virtual void emitWinCFIEndChained(SMLoc Loc = SMLoc()); + virtual void emitWinCFIPushReg(MCRegister Register, SMLoc Loc = SMLoc()); + virtual void emitWinCFISetFrame(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); - virtual void EmitWinCFIAllocStack(unsigned Size, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISaveReg(MCRegister Register, unsigned Offset, + virtual void emitWinCFIAllocStack(unsigned Size, SMLoc Loc = SMLoc()); + virtual void emitWinCFISaveReg(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); - virtual void EmitWinCFISaveXMM(MCRegister Register, unsigned Offset, + virtual void emitWinCFISaveXMM(MCRegister Register, unsigned Offset, SMLoc Loc = SMLoc()); - virtual void EmitWinCFIPushFrame(bool Code, SMLoc Loc = SMLoc()); - virtual void EmitWinCFIEndProlog(SMLoc Loc = SMLoc()); - virtual void EmitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except, + virtual void emitWinCFIPushFrame(bool Code, SMLoc Loc = SMLoc()); + virtual void emitWinCFIEndProlog(SMLoc Loc = SMLoc()); + virtual void emitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except, SMLoc Loc = SMLoc()); - virtual void EmitWinEHHandlerData(SMLoc Loc = SMLoc()); + virtual void emitWinEHHandlerData(SMLoc Loc = SMLoc()); virtual void emitCGProfileEntry(const MCSymbolRefExpr *From, const MCSymbolRefExpr *To, uint64_t Count); @@ -1099,7 +1119,7 @@ public: /// Streamer specific finalization. virtual void finishImpl(); /// Finish emission of machine code. - void Finish(SMLoc EndLoc = SMLoc()); + void finish(SMLoc EndLoc = SMLoc()); virtual bool mayHaveInstructions(MCSection &Sec) const { return true; } diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h index 839a3bd85829..e1f0a86141e3 100644 --- a/llvm/include/llvm/MC/MCSubtargetInfo.h +++ b/llvm/include/llvm/MC/MCSubtargetInfo.h @@ -14,12 +14,13 @@ #define LLVM_MC_MCSUBTARGETINFO_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/MC/MCSchedule.h" #include "llvm/MC/SubtargetFeature.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <string> diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h index d8fc4505d446..91ef6ee31d8d 100644 --- a/llvm/include/llvm/MC/MCSymbol.h +++ b/llvm/include/llvm/MC/MCSymbol.h @@ -14,7 +14,7 @@ #define LLVM_MC_MCSYMBOL_H #include "llvm/ADT/PointerIntPair.h" -#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringMapEntry.h" #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCFragment.h" diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h index 5a4852e0e895..5eab32cb5c12 100644 --- a/llvm/include/llvm/MC/MCSymbolWasm.h +++ b/llvm/include/llvm/MC/MCSymbolWasm.h @@ -86,9 +86,9 @@ public: bool omitFromLinkingSection() const { return OmitFromLinkingSection; } void setOmitFromLinkingSection() { OmitFromLinkingSection = true; } - bool hasImportModule() const { return ImportModule.hasValue(); } + bool hasImportModule() const { return ImportModule.has_value(); } StringRef getImportModule() const { - if (ImportModule.hasValue()) + if (ImportModule) return ImportModule.getValue(); // Use a default module name of "env" for now, for compatibility with // existing tools. @@ -98,15 +98,15 @@ public: } void setImportModule(StringRef Name) { ImportModule = Name; } - bool hasImportName() const { return ImportName.hasValue(); } + bool hasImportName() const { return ImportName.has_value(); } StringRef getImportName() const { - if (ImportName.hasValue()) + if (ImportName) return ImportName.getValue(); return getName(); } void setImportName(StringRef Name) { ImportName = Name; } - bool hasExportName() const { return ExportName.hasValue(); } + bool hasExportName() const { return ExportName.has_value(); } StringRef getExportName() const { return ExportName.getValue(); } void setExportName(StringRef Name) { ExportName = Name; } @@ -129,12 +129,12 @@ public: void setSignature(wasm::WasmSignature *Sig) { Signature = Sig; } const wasm::WasmGlobalType &getGlobalType() const { - assert(GlobalType.hasValue()); + assert(GlobalType); return GlobalType.getValue(); } void setGlobalType(wasm::WasmGlobalType GT) { GlobalType = GT; } - bool hasTableType() const { return TableType.hasValue(); } + bool hasTableType() const { return TableType.has_value(); } const wasm::WasmTableType &getTableType() const { assert(hasTableType()); return TableType.getValue(); diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h index 752e1e7bba0f..2ec265e66300 100644 --- a/llvm/include/llvm/MC/MCSymbolXCOFF.h +++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h @@ -39,8 +39,7 @@ public: }; XCOFF::StorageClass getStorageClass() const { - assert(StorageClass.hasValue() && - "StorageClass not set on XCOFF MCSymbol."); + assert(StorageClass && "StorageClass not set on XCOFF MCSymbol."); return StorageClass.getValue(); } diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h index db50dc6749e2..9c906cdc90d0 100644 --- a/llvm/include/llvm/MC/MCTargetOptions.h +++ b/llvm/include/llvm/MC/MCTargetOptions.h @@ -31,6 +31,12 @@ enum class DebugCompressionType { Z, ///< zlib style complession }; +enum class EmitDwarfUnwindType { + Always, // Always emit dwarf unwind + NoCompactUnwind, // Only emit if compact unwind isn't available + Default, // Default behavior is based on the target +}; + class StringRef; class MCTargetOptions { @@ -47,7 +53,6 @@ public: bool MCNoDeprecatedWarn : 1; bool MCNoTypeCheck : 1; bool MCSaveTempLabels : 1; - bool MCUseDwarfDirectory : 1; bool MCIncrementalLinkerCompatible : 1; bool ShowMCEncoding : 1; bool ShowMCInst : 1; @@ -57,8 +62,22 @@ public: bool PreserveAsmComments : 1; bool Dwarf64 : 1; + + EmitDwarfUnwindType EmitDwarfUnwind; + int DwarfVersion = 0; + enum DwarfDirectory { + // Force disable + DisableDwarfDirectory, + // Force enable, for assemblers that support + // `.file fileno directory filename' syntax + EnableDwarfDirectory, + // Default is based on the target + DefaultDwarfDirectory + }; + DwarfDirectory MCUseDwarfDirectory; + std::string ABIName; std::string AssemblyLanguage; std::string SplitDwarfFile; diff --git a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h index 189484198916..d51e740177f7 100644 --- a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -20,6 +20,7 @@ namespace llvm { class MCTargetOptions; +enum class EmitDwarfUnwindType; namespace mc { @@ -32,6 +33,8 @@ int getDwarfVersion(); bool getDwarf64(); +EmitDwarfUnwindType getEmitDwarfUnwind(); + bool getShowMCInst(); bool getFatalWarnings(); diff --git a/llvm/include/llvm/MC/MCValue.h b/llvm/include/llvm/MC/MCValue.h index 37feee4c9ea8..37265d72c9df 100644 --- a/llvm/include/llvm/MC/MCValue.h +++ b/llvm/include/llvm/MC/MCValue.h @@ -15,7 +15,6 @@ #include "llvm/MC/MCExpr.h" #include "llvm/Support/DataTypes.h" -#include <cassert> namespace llvm { class raw_ostream; diff --git a/llvm/include/llvm/MC/MCWin64EH.h b/llvm/include/llvm/MC/MCWin64EH.h index 065161d1759e..622a666b78dd 100644 --- a/llvm/include/llvm/MC/MCWin64EH.h +++ b/llvm/include/llvm/MC/MCWin64EH.h @@ -57,13 +57,19 @@ public: bool HandlerData) const override; }; -class ARM64UnwindEmitter : public WinEH::UnwindEmitter { +class ARMUnwindEmitter : public WinEH::UnwindEmitter { public: void Emit(MCStreamer &Streamer) const override; void EmitUnwindInfo(MCStreamer &Streamer, WinEH::FrameInfo *FI, bool HandlerData) const override; }; +class ARM64UnwindEmitter : public WinEH::UnwindEmitter { +public: + void Emit(MCStreamer &Streamer) const override; + void EmitUnwindInfo(MCStreamer &Streamer, WinEH::FrameInfo *FI, + bool HandlerData) const override; +}; } } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h index af1ed6faf753..0778c4d52c5e 100644 --- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h +++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h @@ -45,15 +45,15 @@ public: void emitThumbFunc(MCSymbol *Func) override; bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override; void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override; - void BeginCOFFSymbolDef(MCSymbol const *Symbol) override; - void EmitCOFFSymbolStorageClass(int StorageClass) override; - void EmitCOFFSymbolType(int Type) override; - void EndCOFFSymbolDef() override; - void EmitCOFFSafeSEH(MCSymbol const *Symbol) override; - void EmitCOFFSymbolIndex(MCSymbol const *Symbol) override; - void EmitCOFFSectionIndex(MCSymbol const *Symbol) override; - void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) override; - void EmitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset) override; + void beginCOFFSymbolDef(MCSymbol const *Symbol) override; + void emitCOFFSymbolStorageClass(int StorageClass) override; + void emitCOFFSymbolType(int Type) override; + void endCOFFSymbolDef() override; + void emitCOFFSafeSEH(MCSymbol const *Symbol) override; + void emitCOFFSymbolIndex(MCSymbol const *Symbol) override; + void emitCOFFSectionIndex(MCSymbol const *Symbol) override; + void emitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) override; + void emitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset) override; void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment) override; void emitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, @@ -64,7 +64,7 @@ public: void emitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment) override; void emitIdent(StringRef IdentString) override; - void EmitWinEHHandlerData(SMLoc Loc) override; + void emitWinEHHandlerData(SMLoc Loc) override; void emitCGProfileEntry(const MCSymbolRefExpr *From, const MCSymbolRefExpr *To, uint64_t Count) override; void finishImpl() override; diff --git a/llvm/include/llvm/MC/MCWinEH.h b/llvm/include/llvm/MC/MCWinEH.h index 5688255810d0..c16396ea5e71 100644 --- a/llvm/include/llvm/MC/MCWinEH.h +++ b/llvm/include/llvm/MC/MCWinEH.h @@ -50,11 +50,17 @@ struct FrameInfo { bool HandlesUnwind = false; bool HandlesExceptions = false; bool EmitAttempted = false; + bool Fragment = false; int LastFrameInst = -1; const FrameInfo *ChainedParent = nullptr; std::vector<Instruction> Instructions; - MapVector<MCSymbol*, std::vector<Instruction>> EpilogMap; + struct Epilog { + std::vector<Instruction> Instructions; + unsigned Condition; + MCSymbol *End; + }; + MapVector<MCSymbol *, Epilog> EpilogMap; FrameInfo() = default; FrameInfo(const MCSymbol *Function, const MCSymbol *BeginFuncEHLabel) @@ -68,7 +74,7 @@ struct FrameInfo { if (!Instructions.empty()) return false; for (const auto &E : EpilogMap) - if (!E.second.empty()) + if (!E.second.Instructions.empty()) return false; return true; } diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h index 5fc2efbe5284..3faa03fa69e9 100644 --- a/llvm/include/llvm/MC/MCXCOFFStreamer.h +++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h @@ -32,6 +32,10 @@ public: void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol, MCSymbolAttr Linkage, MCSymbolAttr Visibility) override; + void emitXCOFFRefDirective(StringRef Name) override { + report_fatal_error("emitXCOFFRefDirective is not implemented yet on object" + "generation path"); + } void emitXCOFFRenameDirective(const MCSymbol *Name, StringRef Rename) override { report_fatal_error("emitXCOFFRenameDirective is not implemented yet on " diff --git a/llvm/include/llvm/MC/SectionKind.h b/llvm/include/llvm/MC/SectionKind.h index 0fd86cc457de..61e400fe9ede 100644 --- a/llvm/include/llvm/MC/SectionKind.h +++ b/llvm/include/llvm/MC/SectionKind.h @@ -24,6 +24,10 @@ class SectionKind { /// Metadata - Debug info sections or other metadata. Metadata, + /// Exclude - This section will be excluded from the final executable or + /// shared library. Only valid for ELF / COFF targets. + Exclude, + /// Text - Text section, used for functions and other executable code. Text, @@ -118,6 +122,8 @@ public: bool isMetadata() const { return K == Metadata; } + bool isExclude() const { return K == Exclude; } + bool isText() const { return K == Text || K == ExecuteOnly; } bool isExecuteOnly() const { return K == ExecuteOnly; } @@ -180,6 +186,7 @@ private: public: static SectionKind getMetadata() { return get(Metadata); } + static SectionKind getExclude() { return get(Exclude); } static SectionKind getText() { return get(Text); } static SectionKind getExecuteOnly() { return get(ExecuteOnly); } static SectionKind getReadOnly() { return get(ReadOnly); } diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index 3f9c91be05d3..42133f3f7726 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -85,7 +85,6 @@ public: void write(raw_ostream &OS) const; void write(uint8_t *Buf) const; -private: bool isFinalized() const { return Finalized; } }; diff --git a/llvm/include/llvm/MC/SubtargetFeature.h b/llvm/include/llvm/MC/SubtargetFeature.h index 032e2a7df1f2..799912d4bacb 100644 --- a/llvm/include/llvm/MC/SubtargetFeature.h +++ b/llvm/include/llvm/MC/SubtargetFeature.h @@ -17,11 +17,10 @@ #ifndef LLVM_MC_SUBTARGETFEATURE_H #define LLVM_MC_SUBTARGETFEATURE_H -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/MathExtras.h" #include <array> -#include <bitset> #include <initializer_list> #include <string> #include <vector> diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h index da9a9269edbf..eeac559f81b1 100644 --- a/llvm/include/llvm/MC/TargetRegistry.h +++ b/llvm/include/llvm/MC/TargetRegistry.h @@ -27,7 +27,6 @@ #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <iterator> @@ -56,13 +55,12 @@ class MCTargetAsmParser; class MCTargetOptions; class MCTargetStreamer; class raw_ostream; -class raw_pwrite_stream; class TargetMachine; class TargetOptions; namespace mca { class CustomBehaviour; class InstrPostProcess; -class SourceMgr; +struct SourceMgr; } // namespace mca MCStreamer *createNullStreamer(MCContext &Ctx); @@ -111,6 +109,16 @@ MCStreamer *createXCOFFStreamer(MCContext &Ctx, std::unique_ptr<MCObjectWriter> &&OW, std::unique_ptr<MCCodeEmitter> &&CE, bool RelaxAll); +MCStreamer *createSPIRVStreamer(MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + std::unique_ptr<MCObjectWriter> &&OW, + std::unique_ptr<MCCodeEmitter> &&CE, + bool RelaxAll); +MCStreamer *createDXContainerStreamer(MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + std::unique_ptr<MCObjectWriter> &&OW, + std::unique_ptr<MCCodeEmitter> &&CE, + bool RelaxAll); MCRelocationInfo *createMCRelocationInfo(const Triple &TT, MCContext &Ctx); @@ -177,7 +185,6 @@ public: const MCInstrInfo &MII, const MCRegisterInfo &MRI); using MCCodeEmitterCtorTy = MCCodeEmitter *(*)(const MCInstrInfo &II, - const MCRegisterInfo &MRI, MCContext &Ctx); using ELFStreamerCtorTy = MCStreamer *(*)(const Triple &T, MCContext &Ctx, @@ -204,6 +211,17 @@ public: std::unique_ptr<MCAsmBackend> &&TAB, std::unique_ptr<MCObjectWriter> &&OW, std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll); + using SPIRVStreamerCtorTy = + MCStreamer *(*)(const Triple &T, MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + std::unique_ptr<MCObjectWriter> &&OW, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll); + + using DXContainerStreamerCtorTy = + MCStreamer *(*)(const Triple &T, MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + std::unique_ptr<MCObjectWriter> &&OW, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll); using NullTargetStreamerCtorTy = MCTargetStreamer *(*)(MCStreamer &S); using AsmTargetStreamerCtorTy = MCTargetStreamer *(*)( @@ -305,6 +323,8 @@ private: ELFStreamerCtorTy ELFStreamerCtorFn = nullptr; WasmStreamerCtorTy WasmStreamerCtorFn = nullptr; XCOFFStreamerCtorTy XCOFFStreamerCtorFn = nullptr; + SPIRVStreamerCtorTy SPIRVStreamerCtorFn = nullptr; + DXContainerStreamerCtorTy DXContainerStreamerCtorFn = nullptr; /// Construction function for this target's null TargetStreamer, if /// registered (default = nullptr). @@ -508,11 +528,10 @@ public: /// createMCCodeEmitter - Create a target specific code emitter. MCCodeEmitter *createMCCodeEmitter(const MCInstrInfo &II, - const MCRegisterInfo &MRI, MCContext &Ctx) const { if (!MCCodeEmitterCtorFn) return nullptr; - return MCCodeEmitterCtorFn(II, MRI, Ctx); + return MCCodeEmitterCtorFn(II, Ctx); } /// Create a target specific MCStreamer. @@ -576,6 +595,22 @@ public: S = createXCOFFStreamer(Ctx, std::move(TAB), std::move(OW), std::move(Emitter), RelaxAll); break; + case Triple::SPIRV: + if (SPIRVStreamerCtorFn) + S = SPIRVStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); + else + S = createSPIRVStreamer(Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); + break; + case Triple::DXContainer: + if (DXContainerStreamerCtorFn) + S = DXContainerStreamerCtorFn(T, Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); + else + S = createDXContainerStreamer(Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); + break; } if (ObjectTargetStreamerCtorFn) ObjectTargetStreamerCtorFn(*S, STI); @@ -956,6 +991,14 @@ struct TargetRegistry { T.ELFStreamerCtorFn = Fn; } + static void RegisterSPIRVStreamer(Target &T, Target::SPIRVStreamerCtorTy Fn) { + T.SPIRVStreamerCtorFn = Fn; + } + + static void RegisterDXContainerStreamer(Target &T, Target::DXContainerStreamerCtorTy Fn) { + T.DXContainerStreamerCtorFn = Fn; + } + static void RegisterWasmStreamer(Target &T, Target::WasmStreamerCtorTy Fn) { T.WasmStreamerCtorFn = Fn; } @@ -1362,7 +1405,6 @@ template <class MCCodeEmitterImpl> struct RegisterMCCodeEmitter { private: static MCCodeEmitter *Allocator(const MCInstrInfo & /*II*/, - const MCRegisterInfo & /*MRI*/, MCContext & /*Ctx*/) { return new MCCodeEmitterImpl(); } diff --git a/llvm/include/llvm/MCA/CustomBehaviour.h b/llvm/include/llvm/MCA/CustomBehaviour.h index c4be5312ea19..527dc766b739 100644 --- a/llvm/include/llvm/MCA/CustomBehaviour.h +++ b/llvm/include/llvm/MCA/CustomBehaviour.h @@ -49,6 +49,11 @@ public: /// scheduling model. virtual void postProcessInstruction(std::unique_ptr<Instruction> &Inst, const MCInst &MCI) {} + + // The resetState() method gets invoked at the beginning of each code region + // so that targets that override this function can clear any state that they + // have left from the previous code region. + virtual void resetState() {} }; /// Class which can be overriden by targets to enforce instruction diff --git a/llvm/include/llvm/MCA/IncrementalSourceMgr.h b/llvm/include/llvm/MCA/IncrementalSourceMgr.h new file mode 100644 index 000000000000..d91cc5f23311 --- /dev/null +++ b/llvm/include/llvm/MCA/IncrementalSourceMgr.h @@ -0,0 +1,92 @@ +//===---------------- IncrementalSourceMgr.h --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file contains IncrementalSourceMgr, an implementation of SourceMgr +/// that allows users to add new instructions incrementally / dynamically. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MCA_INCREMENTALSOURCEMGR_H +#define LLVM_MCA_INCREMENTALSOURCEMGR_H + +#include "llvm/MCA/SourceMgr.h" +#include <deque> + +namespace llvm { +namespace mca { + +/// An implementation of \a SourceMgr that allows users to add new instructions +/// incrementally / dynamically. +/// Note that this SourceMgr takes ownership of all \a mca::Instruction. +class IncrementalSourceMgr : public SourceMgr { + /// Owner of all mca::Instruction instances. Note that we use std::deque here + /// to have a better throughput, in comparison to std::vector or + /// llvm::SmallVector, as they usually pay a higher re-allocation cost when + /// there is a large number of instructions. + std::deque<UniqueInst> InstStorage; + + /// Instructions that are ready to be used. Each of them is a pointer of an + /// \a UniqueInst inside InstStorage. + std::deque<Instruction *> Staging; + + /// Current instruction index. + unsigned TotalCounter; + + /// End-of-stream flag. + bool EOS; + + /// Called when an instruction is no longer needed. + using InstFreedCallback = llvm::function_ref<void(Instruction *)>; + InstFreedCallback InstFreedCB; + +public: + IncrementalSourceMgr() : TotalCounter(0U), EOS(false) {} + + void clear(); + + /// Set a callback that is invoked when a mca::Instruction is + /// no longer needed. This is usually used for recycling the + /// instruction. + void setOnInstFreedCallback(InstFreedCallback CB) { InstFreedCB = CB; } + + ArrayRef<UniqueInst> getInstructions() const override { + llvm_unreachable("Not applicable"); + } + + bool hasNext() const override { return !Staging.empty(); } + bool isEnd() const override { return EOS; } + + SourceRef peekNext() const override { + assert(hasNext()); + return SourceRef(TotalCounter, *Staging.front()); + } + + /// Add a new instruction. + void addInst(UniqueInst &&Inst) { + InstStorage.emplace_back(std::move(Inst)); + Staging.push_back(InstStorage.back().get()); + } + + /// Add a recycled instruction. + void addRecycledInst(Instruction *Inst) { Staging.push_back(Inst); } + + void updateNext() override; + + /// Mark the end of instruction stream. + void endOfStream() { EOS = true; } + +#ifndef NDEBUG + /// Print statistic about instruction recycling stats. + void printStatistic(raw_ostream &OS); +#endif +}; + +} // end namespace mca +} // end namespace llvm + +#endif // LLVM_MCA_INCREMENTALSOURCEMGR_H diff --git a/llvm/include/llvm/MCA/InstrBuilder.h b/llvm/include/llvm/MCA/InstrBuilder.h index 04b5cf590d70..92b92a515db9 100644 --- a/llvm/include/llvm/MCA/InstrBuilder.h +++ b/llvm/include/llvm/MCA/InstrBuilder.h @@ -14,6 +14,7 @@ #ifndef LLVM_MCA_INSTRBUILDER_H #define LLVM_MCA_INSTRBUILDER_H +#include "llvm/ADT/STLExtras.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" @@ -25,6 +26,27 @@ namespace llvm { namespace mca { +class RecycledInstErr : public ErrorInfo<RecycledInstErr> { + Instruction *RecycledInst; + +public: + static char ID; + + explicit RecycledInstErr(Instruction *Inst) : RecycledInst(Inst) {} + // Always need to carry an Instruction + RecycledInstErr() = delete; + + Instruction *getInst() const { return RecycledInst; } + + void log(raw_ostream &OS) const override { + OS << "Instruction is recycled\n"; + } + + std::error_code convertToErrorCode() const override { + return llvm::inconvertibleErrorCode(); + } +}; + /// A builder class that knows how to construct Instruction objects. /// /// Every llvm-mca Instruction is described by an object of class InstrDesc. @@ -48,6 +70,10 @@ class InstrBuilder { bool FirstCallInst; bool FirstReturnInst; + using InstRecycleCallback = + llvm::function_ref<Instruction *(const InstrDesc &)>; + InstRecycleCallback InstRecycleCB; + Expected<const InstrDesc &> createInstrDescImpl(const MCInst &MCI); Expected<const InstrDesc &> getOrCreateInstrDesc(const MCInst &MCI); @@ -69,6 +95,10 @@ public: FirstReturnInst = true; } + /// Set a callback which is invoked to retrieve a recycled mca::Instruction + /// or null if there isn't any. + void setInstRecycleCallback(InstRecycleCallback CB) { InstRecycleCB = CB; } + Expected<std::unique_ptr<Instruction>> createInstruction(const MCInst &MCI); }; } // namespace mca diff --git a/llvm/include/llvm/MCA/Instruction.h b/llvm/include/llvm/MCA/Instruction.h index 33e3c8a2e630..86f2d7ade161 100644 --- a/llvm/include/llvm/MCA/Instruction.h +++ b/llvm/include/llvm/MCA/Instruction.h @@ -472,17 +472,15 @@ struct InstrDesc { // subtarget when computing the reciprocal throughput. unsigned SchedClassID; - unsigned MayLoad : 1; - unsigned MayStore : 1; - unsigned HasSideEffects : 1; - unsigned BeginGroup : 1; - unsigned EndGroup : 1; - unsigned RetireOOO : 1; - // True if all buffered resources are in-order, and there is at least one // buffer which is a dispatch hazard (BufferSize = 0). unsigned MustIssueImmediately : 1; + // True if the corresponding mca::Instruction can be recycled. Currently only + // instructions that are neither variadic nor have any variant can be + // recycled. + unsigned IsRecyclable : 1; + // A zero latency instruction doesn't consume any scheduler resources. bool isZeroLatency() const { return !MaxLatency && Resources.empty(); } @@ -518,8 +516,16 @@ class InstructionBase { unsigned Opcode; // Flags used by the LSUnit. - bool IsALoadBarrier; - bool IsAStoreBarrier; + bool IsALoadBarrier : 1; + bool IsAStoreBarrier : 1; + // Flags copied from the InstrDesc and potentially modified by + // CustomBehaviour or (more likely) InstrPostProcess. + bool MayLoad : 1; + bool MayStore : 1; + bool HasSideEffects : 1; + bool BeginGroup : 1; + bool EndGroup : 1; + bool RetireOOO : 1; public: InstructionBase(const InstrDesc &D, const unsigned Opcode) @@ -568,7 +574,23 @@ public: // Returns true if this instruction is a candidate for move elimination. bool isOptimizableMove() const { return IsOptimizableMove; } void setOptimizableMove() { IsOptimizableMove = true; } - bool isMemOp() const { return Desc.MayLoad || Desc.MayStore; } + void clearOptimizableMove() { IsOptimizableMove = false; } + bool isMemOp() const { return MayLoad || MayStore; } + + // Getters and setters for general instruction flags. + void setMayLoad(bool newVal) { MayLoad = newVal; } + void setMayStore(bool newVal) { MayStore = newVal; } + void setHasSideEffects(bool newVal) { HasSideEffects = newVal; } + void setBeginGroup(bool newVal) { BeginGroup = newVal; } + void setEndGroup(bool newVal) { EndGroup = newVal; } + void setRetireOOO(bool newVal) { RetireOOO = newVal; } + + bool getMayLoad() const { return MayLoad; } + bool getMayStore() const { return MayStore; } + bool getHasSideEffects() const { return HasSideEffects; } + bool getBeginGroup() const { return BeginGroup; } + bool getEndGroup() const { return EndGroup; } + bool getRetireOOO() const { return RetireOOO; } }; /// An instruction propagated through the simulated instruction pipeline. @@ -628,6 +650,8 @@ public: UsedBuffers(D.UsedBuffers), CriticalRegDep(), CriticalMemDep(), CriticalResourceMask(0), IsEliminated(false) {} + void reset(); + unsigned getRCUTokenID() const { return RCUTokenID; } unsigned getLSUTokenID() const { return LSUTokenID; } void setLSUTokenID(unsigned LSUTok) { LSUTokenID = LSUTok; } @@ -657,6 +681,7 @@ public: bool updateDispatched(); bool updatePending(); + bool isInvalid() const { return Stage == IS_INVALID; } bool isDispatched() const { return Stage == IS_DISPATCHED; } bool isPending() const { return Stage == IS_PENDING; } bool isReady() const { return Stage == IS_READY; } diff --git a/llvm/include/llvm/MCA/Pipeline.h b/llvm/include/llvm/MCA/Pipeline.h index 0ac988c52dc1..92c3836124ad 100644 --- a/llvm/include/llvm/MCA/Pipeline.h +++ b/llvm/include/llvm/MCA/Pipeline.h @@ -51,6 +51,13 @@ class Pipeline { Pipeline(const Pipeline &P) = delete; Pipeline &operator=(const Pipeline &P) = delete; + enum class State { + Created, // Pipeline was just created. The default state. + Started, // Pipeline has started running. + Paused // Pipeline is paused. + }; + State CurrentState; + /// An ordered list of stages that define this instruction pipeline. SmallVector<std::unique_ptr<Stage>, 8> Stages; std::set<HWEventListener *> Listeners; @@ -62,13 +69,16 @@ class Pipeline { void notifyCycleEnd(); public: - Pipeline() : Cycles(0) {} + Pipeline() : CurrentState(State::Created), Cycles(0) {} void appendStage(std::unique_ptr<Stage> S); /// Returns the total number of simulated cycles. Expected<unsigned> run(); void addEventListener(HWEventListener *Listener); + + /// Returns whether the pipeline is currently paused. + bool isPaused() const { return CurrentState == State::Paused; } }; } // namespace mca } // namespace llvm diff --git a/llvm/include/llvm/MCA/SourceMgr.h b/llvm/include/llvm/MCA/SourceMgr.h index e844171bdcab..16a60d1116ad 100644 --- a/llvm/include/llvm/MCA/SourceMgr.h +++ b/llvm/include/llvm/MCA/SourceMgr.h @@ -6,9 +6,8 @@ // //===----------------------------------------------------------------------===// /// \file -/// This file implements class SourceMgr. Class SourceMgr abstracts the input -/// code sequence (a sequence of MCInst), and assings unique identifiers to -/// every instruction in the sequence. +/// This file contains abstract class SourceMgr and the default implementation, +/// CircularSourceMgr. /// //===----------------------------------------------------------------------===// @@ -25,30 +24,62 @@ namespace mca { // prevent compiler error C2139 about intrinsic type trait '__is_assignable'. typedef std::pair<unsigned, const Instruction &> SourceRef; -class SourceMgr { +/// Abstracting the input code sequence (a sequence of MCInst) and assigning +/// unique identifiers to every instruction in the sequence. +struct SourceMgr { using UniqueInst = std::unique_ptr<Instruction>; + + /// Provides a fixed range of \a UniqueInst to iterate. + virtual ArrayRef<UniqueInst> getInstructions() const = 0; + + /// (Fixed) Number of \a UniqueInst. Returns the size of + /// \a getInstructions by default. + virtual size_t size() const { return getInstructions().size(); } + + /// Whether there is any \a SourceRef to inspect / peek next. + /// Note that returning false from this doesn't mean the instruction + /// stream has ended. + virtual bool hasNext() const = 0; + + /// Whether the instruction stream has eneded. + virtual bool isEnd() const = 0; + + /// The next \a SourceRef. + virtual SourceRef peekNext() const = 0; + + /// Advance to the next \a SourceRef. + virtual void updateNext() = 0; + + virtual ~SourceMgr() {} +}; + +/// The default implementation of \a SourceMgr. It always takes a fixed number +/// of instructions and provides an option to loop the given sequence for a +/// certain iterations. +class CircularSourceMgr : public SourceMgr { ArrayRef<UniqueInst> Sequence; unsigned Current; const unsigned Iterations; static const unsigned DefaultIterations = 100; public: - SourceMgr(ArrayRef<UniqueInst> S, unsigned Iter) - : Sequence(S), Current(0), Iterations(Iter ? Iter : DefaultIterations) {} + CircularSourceMgr(ArrayRef<UniqueInst> S, unsigned Iter) + : Sequence(S), Current(0U), Iterations(Iter ? Iter : DefaultIterations) {} + + ArrayRef<UniqueInst> getInstructions() const override { return Sequence; } unsigned getNumIterations() const { return Iterations; } - unsigned size() const { return Sequence.size(); } - bool hasNext() const { return Current < (Iterations * Sequence.size()); } - void updateNext() { ++Current; } + bool hasNext() const override { + return Current < (Iterations * Sequence.size()); + } + bool isEnd() const override { return !hasNext(); } - SourceRef peekNext() const { + SourceRef peekNext() const override { assert(hasNext() && "Already at end of sequence!"); return SourceRef(Current, *Sequence[Current % Sequence.size()]); } - using const_iterator = ArrayRef<UniqueInst>::const_iterator; - const_iterator begin() const { return Sequence.begin(); } - const_iterator end() const { return Sequence.end(); } + void updateNext() override { ++Current; } }; } // namespace mca diff --git a/llvm/include/llvm/MCA/Stages/EntryStage.h b/llvm/include/llvm/MCA/Stages/EntryStage.h index 4c50838bef4b..fb1244aa1933 100644 --- a/llvm/include/llvm/MCA/Stages/EntryStage.h +++ b/llvm/include/llvm/MCA/Stages/EntryStage.h @@ -30,7 +30,7 @@ class EntryStage final : public Stage { unsigned NumRetired; // Updates the program counter, and sets 'CurrentInstruction'. - void getNextInstruction(); + Error getNextInstruction(); EntryStage(const EntryStage &Other) = delete; EntryStage &operator=(const EntryStage &Other) = delete; @@ -42,6 +42,7 @@ public: bool hasWorkToComplete() const override; Error execute(InstRef &IR) override; Error cycleStart() override; + Error cycleResume() override; Error cycleEnd() override; }; diff --git a/llvm/include/llvm/MCA/Stages/Stage.h b/llvm/include/llvm/MCA/Stages/Stage.h index 84868e89ac29..2477b9b3d69c 100644 --- a/llvm/include/llvm/MCA/Stages/Stage.h +++ b/llvm/include/llvm/MCA/Stages/Stage.h @@ -48,6 +48,9 @@ public: /// phase to prepare for the executions during the cycle. virtual Error cycleStart() { return ErrorSuccess(); } + /// Called after the pipeline is resumed from pausing state. + virtual Error cycleResume() { return ErrorSuccess(); } + /// Called once at the end of each cycle. virtual Error cycleEnd() { return ErrorSuccess(); } @@ -82,6 +85,16 @@ public: } }; +/// This is actually not an error but a marker to indicate that +/// the instruction stream is paused. +struct InstStreamPause : public ErrorInfo<InstStreamPause> { + static char ID; + + std::error_code convertToErrorCode() const override { + return llvm::inconvertibleErrorCode(); + } + void log(raw_ostream &OS) const override { OS << "Stream is paused"; } +}; } // namespace mca } // namespace llvm #endif // LLVM_MCA_STAGES_STAGE_H diff --git a/llvm/include/llvm/ObjCopy/COFF/COFFConfig.h b/llvm/include/llvm/ObjCopy/COFF/COFFConfig.h new file mode 100644 index 000000000000..29d56d75698b --- /dev/null +++ b/llvm/include/llvm/ObjCopy/COFF/COFFConfig.h @@ -0,0 +1,27 @@ +//===- COFFConfig.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_COFF_COFFCONFIG_H +#define LLVM_OBJCOPY_COFF_COFFCONFIG_H + +#include "llvm/ADT/Optional.h" + +namespace llvm { +namespace objcopy { + +// Coff specific configuration for copying/stripping a single file. +struct COFFConfig { + Optional<unsigned> Subsystem; + Optional<unsigned> MajorSubsystemVersion; + Optional<unsigned> MinorSubsystemVersion; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_COFF_COFFCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h b/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h new file mode 100644 index 000000000000..d9043d6c5d01 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h @@ -0,0 +1,36 @@ +//===- COFFObjcopy.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_COFF_COFFOBJCOPY_H +#define LLVM_OBJCOPY_COFF_COFFOBJCOPY_H + +namespace llvm { +class Error; +class raw_ostream; + +namespace object { +class COFFObjectFile; +} // end namespace object + +namespace objcopy { +struct CommonConfig; +struct COFFConfig; + +namespace coff { + +/// Apply the transformations described by \p Config and \p COFFConfig +/// to \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const CommonConfig &Config, const COFFConfig &, + object::COFFObjectFile &In, raw_ostream &Out); + +} // end namespace coff +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_COFF_COFFOBJCOPY_H diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h new file mode 100644 index 000000000000..24503caed342 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/CommonConfig.h @@ -0,0 +1,271 @@ +//===- CommonConfig.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_COMMONCONFIG_H +#define LLVM_OBJCOPY_COMMONCONFIG_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/CachedHashString.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Object/ELFTypes.h" +#include "llvm/Support/GlobPattern.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Regex.h" +// Necessary for llvm::DebugCompressionType::None +#include "llvm/Target/TargetOptions.h" +#include <vector> + +namespace llvm { +namespace objcopy { + +enum class FileFormat { + Unspecified, + ELF, + Binary, + IHex, +}; + +// This type keeps track of the machine info for various architectures. This +// lets us map architecture names to ELF types and the e_machine value of the +// ELF file. +struct MachineInfo { + MachineInfo(uint16_t EM, uint8_t ABI, bool Is64, bool IsLittle) + : EMachine(EM), OSABI(ABI), Is64Bit(Is64), IsLittleEndian(IsLittle) {} + // Alternative constructor that defaults to NONE for OSABI. + MachineInfo(uint16_t EM, bool Is64, bool IsLittle) + : MachineInfo(EM, ELF::ELFOSABI_NONE, Is64, IsLittle) {} + // Default constructor for unset fields. + MachineInfo() : MachineInfo(0, 0, false, false) {} + uint16_t EMachine; + uint8_t OSABI; + bool Is64Bit; + bool IsLittleEndian; +}; + +// Flags set by --set-section-flags or --rename-section. Interpretation of these +// is format-specific and not all flags are meaningful for all object file +// formats. This is a bitmask; many section flags may be set. +enum SectionFlag { + SecNone = 0, + SecAlloc = 1 << 0, + SecLoad = 1 << 1, + SecNoload = 1 << 2, + SecReadonly = 1 << 3, + SecDebug = 1 << 4, + SecCode = 1 << 5, + SecData = 1 << 6, + SecRom = 1 << 7, + SecMerge = 1 << 8, + SecStrings = 1 << 9, + SecContents = 1 << 10, + SecShare = 1 << 11, + SecExclude = 1 << 12, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/SecExclude) +}; + +struct SectionRename { + StringRef OriginalName; + StringRef NewName; + Optional<SectionFlag> NewFlags; +}; + +struct SectionFlagsUpdate { + StringRef Name; + SectionFlag NewFlags; +}; + +enum class DiscardType { + None, // Default + All, // --discard-all (-x) + Locals, // --discard-locals (-X) +}; + +enum class MatchStyle { + Literal, // Default for symbols. + Wildcard, // Default for sections, or enabled with --wildcard (-w). + Regex, // Enabled with --regex. +}; + +class NameOrPattern { + StringRef Name; + // Regex is shared between multiple CommonConfig instances. + std::shared_ptr<Regex> R; + std::shared_ptr<GlobPattern> G; + bool IsPositiveMatch = true; + + NameOrPattern(StringRef N) : Name(N) {} + NameOrPattern(std::shared_ptr<Regex> R) : R(R) {} + NameOrPattern(std::shared_ptr<GlobPattern> G, bool IsPositiveMatch) + : G(G), IsPositiveMatch(IsPositiveMatch) {} + +public: + // ErrorCallback is used to handle recoverable errors. An Error returned + // by the callback aborts the parsing and is then returned by this function. + static Expected<NameOrPattern> + create(StringRef Pattern, MatchStyle MS, + llvm::function_ref<Error(Error)> ErrorCallback); + + bool isPositiveMatch() const { return IsPositiveMatch; } + Optional<StringRef> getName() const { + if (!R && !G) + return Name; + return None; + } + bool operator==(StringRef S) const { + return R ? R->match(S) : G ? G->match(S) : Name == S; + } + bool operator!=(StringRef S) const { return !operator==(S); } +}; + +// Matcher that checks symbol or section names against the command line flags +// provided for that option. +class NameMatcher { + DenseSet<CachedHashStringRef> PosNames; + std::vector<NameOrPattern> PosPatterns; + std::vector<NameOrPattern> NegMatchers; + +public: + Error addMatcher(Expected<NameOrPattern> Matcher) { + if (!Matcher) + return Matcher.takeError(); + if (Matcher->isPositiveMatch()) { + if (Optional<StringRef> MaybeName = Matcher->getName()) + PosNames.insert(CachedHashStringRef(*MaybeName)); + else + PosPatterns.push_back(std::move(*Matcher)); + } else { + NegMatchers.push_back(std::move(*Matcher)); + } + return Error::success(); + } + bool matches(StringRef S) const { + return (PosNames.contains(CachedHashStringRef(S)) || + is_contained(PosPatterns, S)) && + !is_contained(NegMatchers, S); + } + bool empty() const { + return PosNames.empty() && PosPatterns.empty() && NegMatchers.empty(); + } +}; + +enum class SymbolFlag { + Global, + Local, + Weak, + Default, + Hidden, + Protected, + File, + Section, + Object, + Function, + IndirectFunction, + Debug, + Constructor, + Warning, + Indirect, + Synthetic, + UniqueObject, +}; + +// Symbol info specified by --add-symbol option. Symbol flags not supported +// by a concrete format should be ignored. +struct NewSymbolInfo { + StringRef SymbolName; + StringRef SectionName; + uint64_t Value = 0; + std::vector<SymbolFlag> Flags; + std::vector<StringRef> BeforeSyms; +}; + +// Specify section name and section body for newly added or updated section. +struct NewSectionInfo { + NewSectionInfo() = default; + NewSectionInfo(StringRef Name, std::unique_ptr<MemoryBuffer> &&Buffer) + : SectionName(Name), SectionData(std::move(Buffer)) {} + + StringRef SectionName; + std::shared_ptr<MemoryBuffer> SectionData; +}; + +// Configuration for copying/stripping a single file. +struct CommonConfig { + // Main input/output options + StringRef InputFilename; + FileFormat InputFormat = FileFormat::Unspecified; + StringRef OutputFilename; + FileFormat OutputFormat = FileFormat::Unspecified; + + // Only applicable when --output-format!=binary (e.g. elf64-x86-64). + Optional<MachineInfo> OutputArch; + + // Advanced options + StringRef AddGnuDebugLink; + // Cached gnu_debuglink's target CRC + uint32_t GnuDebugLinkCRC32; + Optional<StringRef> ExtractPartition; + StringRef SplitDWO; + StringRef SymbolsPrefix; + StringRef AllocSectionsPrefix; + DiscardType DiscardMode = DiscardType::None; + + // Repeated options + std::vector<NewSectionInfo> AddSection; + std::vector<StringRef> DumpSection; + std::vector<NewSectionInfo> UpdateSection; + + // Section matchers + NameMatcher KeepSection; + NameMatcher OnlySection; + NameMatcher ToRemove; + + // Symbol matchers + NameMatcher SymbolsToGlobalize; + NameMatcher SymbolsToKeep; + NameMatcher SymbolsToLocalize; + NameMatcher SymbolsToRemove; + NameMatcher UnneededSymbolsToRemove; + NameMatcher SymbolsToWeaken; + NameMatcher SymbolsToKeepGlobal; + + // Map options + StringMap<SectionRename> SectionsToRename; + StringMap<uint64_t> SetSectionAlignment; + StringMap<SectionFlagsUpdate> SetSectionFlags; + StringMap<StringRef> SymbolsToRename; + + // Symbol info specified by --add-symbol option. + std::vector<NewSymbolInfo> SymbolsToAdd; + + // Boolean options + bool DeterministicArchives = true; + bool ExtractDWO = false; + bool ExtractMainPartition = false; + bool OnlyKeepDebug = false; + bool PreserveDates = false; + bool StripAll = false; + bool StripAllGNU = false; + bool StripDWO = false; + bool StripDebug = false; + bool StripNonAlloc = false; + bool StripSections = false; + bool StripUnneeded = false; + bool Weaken = false; + bool DecompressDebugSections = false; + + DebugCompressionType CompressionType = DebugCompressionType::None; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_COMMONCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/ConfigManager.h b/llvm/include/llvm/ObjCopy/ConfigManager.h new file mode 100644 index 000000000000..2962cf99b270 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/ConfigManager.h @@ -0,0 +1,50 @@ +//===- ConfigManager.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_CONFIGMANAGER_H +#define LLVM_OBJCOPY_CONFIGMANAGER_H + +#include "llvm/ObjCopy/COFF/COFFConfig.h" +#include "llvm/ObjCopy/CommonConfig.h" +#include "llvm/ObjCopy/ELF/ELFConfig.h" +#include "llvm/ObjCopy/MachO/MachOConfig.h" +#include "llvm/ObjCopy/MultiFormatConfig.h" +#include "llvm/ObjCopy/wasm/WasmConfig.h" +#include "llvm/ObjCopy/XCOFF/XCOFFConfig.h" + +namespace llvm { +namespace objcopy { + +struct ConfigManager : public MultiFormatConfig { + virtual ~ConfigManager() {} + + const CommonConfig &getCommonConfig() const override { return Common; } + + Expected<const ELFConfig &> getELFConfig() const override { return ELF; } + + Expected<const COFFConfig &> getCOFFConfig() const override; + + Expected<const MachOConfig &> getMachOConfig() const override; + + Expected<const WasmConfig &> getWasmConfig() const override; + + Expected<const XCOFFConfig &> getXCOFFConfig() const override; + + // All configs. + CommonConfig Common; + ELFConfig ELF; + COFFConfig COFF; + MachOConfig MachO; + WasmConfig Wasm; + XCOFFConfig XCOFF; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_CONFIGMANAGER_H diff --git a/llvm/include/llvm/ObjCopy/ELF/ELFConfig.h b/llvm/include/llvm/ObjCopy/ELF/ELFConfig.h new file mode 100644 index 000000000000..52bc728e36ff --- /dev/null +++ b/llvm/include/llvm/ObjCopy/ELF/ELFConfig.h @@ -0,0 +1,38 @@ +//===- ELFConfig.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_ELF_ELFCONFIG_H +#define LLVM_OBJCOPY_ELF_ELFCONFIG_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Object/ELFTypes.h" +#include <vector> + +namespace llvm { +namespace objcopy { + +// ELF specific configuration for copying/stripping a single file. +struct ELFConfig { + uint8_t NewSymbolVisibility = (uint8_t)ELF::STV_DEFAULT; + + // ELF entry point address expression. The input parameter is an entry point + // address in the input ELF file. The entry address in the output file is + // calculated with EntryExpr(input_address), when either --set-start or + // --change-start is used. + std::function<uint64_t(uint64_t)> EntryExpr; + + bool AllowBrokenLinks = false; + bool KeepFileSymbols = false; + bool LocalizeHidden = false; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_ELF_ELFCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h b/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h new file mode 100644 index 000000000000..552b6fb655f1 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h @@ -0,0 +1,53 @@ +//===- ELFObjcopy.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_ELF_ELFOBJCOPY_H +#define LLVM_OBJCOPY_ELF_ELFOBJCOPY_H + +namespace llvm { +class Error; +class MemoryBuffer; +class raw_ostream; + +namespace object { +class ELFObjectFileBase; +} // end namespace object + +namespace objcopy { +struct CommonConfig; +struct ELFConfig; + +namespace elf { +/// Apply the transformations described by \p Config and \p ELFConfig to +/// \p In, which must represent an IHex file, and writes the result +/// into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnIHex(const CommonConfig &Config, + const ELFConfig &ELFConfig, MemoryBuffer &In, + raw_ostream &Out); + +/// Apply the transformations described by \p Config and \p ELFConfig to +/// \p In, which is treated as a raw binary input, and writes the result +/// into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnRawBinary(const CommonConfig &Config, + const ELFConfig &ELFConfig, MemoryBuffer &In, + raw_ostream &Out); + +/// Apply the transformations described by \p Config and \p ELFConfig to +/// \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const CommonConfig &Config, + const ELFConfig &ELFConfig, + object::ELFObjectFileBase &In, raw_ostream &Out); + +} // end namespace elf +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_ELF_ELFOBJCOPY_H diff --git a/llvm/include/llvm/ObjCopy/MachO/MachOConfig.h b/llvm/include/llvm/ObjCopy/MachO/MachOConfig.h new file mode 100644 index 000000000000..c5f861363297 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/MachO/MachOConfig.h @@ -0,0 +1,46 @@ +//===- MachOConfig.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_MACHO_MACHOCONFIG_H +#define LLVM_OBJCOPY_MACHO_MACHOCONFIG_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" +#include <vector> + +namespace llvm { +namespace objcopy { + +// Mach-O specific configuration for copying/stripping a single file. +struct MachOConfig { + // Repeated options + std::vector<StringRef> RPathToAdd; + std::vector<StringRef> RPathToPrepend; + DenseMap<StringRef, StringRef> RPathsToUpdate; + DenseMap<StringRef, StringRef> InstallNamesToUpdate; + DenseSet<StringRef> RPathsToRemove; + + // install-name-tool's id option + Optional<StringRef> SharedLibId; + + // Segments to remove if they are empty + DenseSet<StringRef> EmptySegmentsToRemove; + + // Boolean options + bool StripSwiftSymbols = false; + bool KeepUndefined = false; + + // install-name-tool's --delete_all_rpaths + bool RemoveAllRpaths = false; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_MACHO_MACHOCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h b/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h new file mode 100644 index 000000000000..73690d7ace8a --- /dev/null +++ b/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h @@ -0,0 +1,45 @@ +//===- MachOObjcopy.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_MACHO_MACHOOBJCOPY_H +#define LLVM_OBJCOPY_MACHO_MACHOOBJCOPY_H + +namespace llvm { +class Error; +class raw_ostream; + +namespace object { +class MachOObjectFile; +class MachOUniversalBinary; +} // end namespace object + +namespace objcopy { +struct CommonConfig; +struct MachOConfig; +class MultiFormatConfig; + +namespace macho { +/// Apply the transformations described by \p Config and \p MachOConfig to +/// \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const CommonConfig &Config, + const MachOConfig &MachOConfig, + object::MachOObjectFile &In, raw_ostream &Out); + +/// Apply the transformations described by \p Config and \p MachOConfig to +/// \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnMachOUniversalBinary( + const MultiFormatConfig &Config, const object::MachOUniversalBinary &In, + raw_ostream &Out); + +} // end namespace macho +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_MACHO_MACHOOBJCOPY_H diff --git a/llvm/include/llvm/ObjCopy/MultiFormatConfig.h b/llvm/include/llvm/ObjCopy/MultiFormatConfig.h new file mode 100644 index 000000000000..180f2f82a908 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/MultiFormatConfig.h @@ -0,0 +1,39 @@ +//===- MultiFormatConfig.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_MULTIFORMATCONFIG_H +#define LLVM_OBJCOPY_MULTIFORMATCONFIG_H + +#include "llvm/Support/Error.h" + +namespace llvm { +namespace objcopy { + +struct CommonConfig; +struct ELFConfig; +struct COFFConfig; +struct MachOConfig; +struct WasmConfig; +struct XCOFFConfig; + +class MultiFormatConfig { +public: + virtual ~MultiFormatConfig() {} + + virtual const CommonConfig &getCommonConfig() const = 0; + virtual Expected<const ELFConfig &> getELFConfig() const = 0; + virtual Expected<const COFFConfig &> getCOFFConfig() const = 0; + virtual Expected<const MachOConfig &> getMachOConfig() const = 0; + virtual Expected<const WasmConfig &> getWasmConfig() const = 0; + virtual Expected<const XCOFFConfig &> getXCOFFConfig() const = 0; +}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_MULTIFORMATCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/ObjCopy.h b/llvm/include/llvm/ObjCopy/ObjCopy.h new file mode 100644 index 000000000000..023814002c72 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/ObjCopy.h @@ -0,0 +1,42 @@ +//===- ObjCopy.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_OBJCOPY_H +#define LLVM_OBJCOPY_OBJCOPY_H + +#include "llvm/Support/Error.h" + +namespace llvm { +class raw_ostream; + +namespace object { +class Archive; +class Binary; +} // end namespace object + +namespace objcopy { +class MultiFormatConfig; + +/// Applies the transformations described by \p Config to +/// each member in archive \p Ar. +/// Writes a result in a file specified by \p Config.OutputFilename. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnArchive(const MultiFormatConfig &Config, + const object::Archive &Ar); + +/// Applies the transformations described by \p Config to \p In and writes +/// the result into \p Out. This function does the dispatch based on the +/// format of the input binary (COFF, ELF, MachO or wasm). +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const MultiFormatConfig &Config, + object::Binary &In, raw_ostream &Out); + +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_OBJCOPY_H diff --git a/llvm/include/llvm/ObjCopy/XCOFF/XCOFFConfig.h b/llvm/include/llvm/ObjCopy/XCOFF/XCOFFConfig.h new file mode 100644 index 000000000000..adaeedc82b73 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/XCOFF/XCOFFConfig.h @@ -0,0 +1,21 @@ +//===- XCOFFConfig.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_XCOFF_XCOFFCONFIG_H +#define LLVM_OBJCOPY_XCOFF_XCOFFCONFIG_H + +namespace llvm { +namespace objcopy { + +// XCOFF specific configuration for copying/stripping a single file. +struct XCOFFConfig {}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_XCOFF_XCOFFCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/XCOFF/XCOFFObjcopy.h b/llvm/include/llvm/ObjCopy/XCOFF/XCOFFObjcopy.h new file mode 100644 index 000000000000..9fc85cb39fa5 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/XCOFF/XCOFFObjcopy.h @@ -0,0 +1,35 @@ +//===- XCOFFObjcopy.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_XCOFF_XCOFFOBJCOPY_H +#define LLVM_OBJCOPY_XCOFF_XCOFFOBJCOPY_H + +namespace llvm { +class Error; +class raw_ostream; + +namespace object { +class XCOFFObjectFile; +} // end namespace object + +namespace objcopy { +struct CommonConfig; +struct XCOFFConfig; + +namespace xcoff { +/// Apply the transformations described by \p Config and \p XCOFFConfig +/// to \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const CommonConfig &Config, const XCOFFConfig &, + object::XCOFFObjectFile &In, raw_ostream &Out); + +} // end namespace xcoff +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_XCOFF_XCOFFOBJCOPY_H diff --git a/llvm/include/llvm/ObjCopy/wasm/WasmConfig.h b/llvm/include/llvm/ObjCopy/wasm/WasmConfig.h new file mode 100644 index 000000000000..56a7055da9a7 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/wasm/WasmConfig.h @@ -0,0 +1,21 @@ +//===- WasmConfig.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_WASM_WASMCONFIG_H +#define LLVM_OBJCOPY_WASM_WASMCONFIG_H + +namespace llvm { +namespace objcopy { + +// Wasm specific configuration for copying/stripping a single file. +struct WasmConfig {}; + +} // namespace objcopy +} // namespace llvm + +#endif // LLVM_OBJCOPY_WASM_WASMCONFIG_H diff --git a/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h b/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h new file mode 100644 index 000000000000..5b4181c22b97 --- /dev/null +++ b/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h @@ -0,0 +1,35 @@ +//===- WasmObjcopy.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJCOPY_WASM_WASMOBJCOPY_H +#define LLVM_OBJCOPY_WASM_WASMOBJCOPY_H + +namespace llvm { +class Error; +class raw_ostream; + +namespace object { +class WasmObjectFile; +} // end namespace object + +namespace objcopy { +struct CommonConfig; +struct WasmConfig; + +namespace wasm { +/// Apply the transformations described by \p Config and \p WasmConfig +/// to \p In and writes the result into \p Out. +/// \returns any Error encountered whilst performing the operation. +Error executeObjcopyOnBinary(const CommonConfig &Config, const WasmConfig &, + object::WasmObjectFile &In, raw_ostream &Out); + +} // end namespace wasm +} // end namespace objcopy +} // end namespace llvm + +#endif // LLVM_OBJCOPY_WASM_WASMOBJCOPY_H diff --git a/llvm/include/llvm/Object/Archive.h b/llvm/include/llvm/Object/Archive.h index b792cbc3d9ac..a36c9bd6163b 100644 --- a/llvm/include/llvm/Object/Archive.h +++ b/llvm/include/llvm/Object/Archive.h @@ -13,7 +13,6 @@ #ifndef LLVM_OBJECT_ARCHIVE_H #define LLVM_OBJECT_ARCHIVE_H -#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/fallible_iterator.h" #include "llvm/ADT/iterator_range.h" @@ -22,7 +21,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <memory> @@ -30,6 +28,9 @@ #include <vector> namespace llvm { + +template <typename T> class Optional; + namespace object { const char ArchiveMagic[] = "!<arch>\n"; @@ -339,6 +340,7 @@ public: Kind kind() const { return (Kind)Format; } bool isThin() const { return IsThin; } + static object::Archive::Kind getDefaultKindForHost(); child_iterator child_begin(Error &Err, bool SkipInternal = true) const; child_iterator child_end() const; @@ -358,7 +360,7 @@ public: // check if a symbol is in the archive Expected<Optional<Child>> findSym(StringRef name) const; - bool isEmpty() const; + virtual bool isEmpty() const; bool hasSymbolTable() const; StringRef getSymbolTable() const { return SymbolTable; } StringRef getStringTable() const { return StringTable; } @@ -390,6 +392,7 @@ private: }; class BigArchive : public Archive { +public: /// Fixed-Length Header. struct FixLenHdr { char Magic[sizeof(BigArchiveMagic) - 1]; ///< Big archive magic string. @@ -410,6 +413,9 @@ public: BigArchive(MemoryBufferRef Source, Error &Err); uint64_t getFirstChildOffset() const override { return FirstChildOffset; } uint64_t getLastChildOffset() const { return LastChildOffset; } + bool isEmpty() const override { + return Data.getBufferSize() == sizeof(FixLenHdr); + }; }; } // end namespace object diff --git a/llvm/include/llvm/Object/ArchiveWriter.h b/llvm/include/llvm/Object/ArchiveWriter.h index 7eaf13e8fb22..6acab45215da 100644 --- a/llvm/include/llvm/Object/ArchiveWriter.h +++ b/llvm/include/llvm/Object/ArchiveWriter.h @@ -26,6 +26,11 @@ struct NewArchiveMember { NewArchiveMember() = default; NewArchiveMember(MemoryBufferRef BufRef); + // Detect the archive format from the object or bitcode file. This helps + // assume the archive format when creating or editing archives in the case + // one isn't explicitly set. + object::Archive::Kind detectKindFromObject() const; + static Expected<NewArchiveMember> getOldMember(const object::Archive::Child &OldMember, bool Deterministic); diff --git a/llvm/include/llvm/Object/Binary.h b/llvm/include/llvm/Object/Binary.h index a8f4437d5dbb..53b299ae8612 100644 --- a/llvm/include/llvm/Object/Binary.h +++ b/llvm/include/llvm/Object/Binary.h @@ -16,9 +16,9 @@ #include "llvm-c/Types.h" #include "llvm/ADT/Triple.h" #include "llvm/Object/Error.h" +#include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" -#include <algorithm> #include <memory> #include <utility> @@ -50,6 +50,8 @@ protected: ID_WinRes, // Windows resource (.res) file. + ID_Offload, // Offloading binary file. + // Object and children. ID_StartObjects, ID_COFF, @@ -133,6 +135,8 @@ public: bool isWasm() const { return TypeID == ID_Wasm; } + bool isOffloadFile() const { return TypeID == ID_Offload; } + bool isCOFFImportFile() const { return TypeID == ID_COFFImportFile; } diff --git a/llvm/include/llvm/Object/COFF.h b/llvm/include/llvm/Object/COFF.h index 3add3811069b..0b6975b9590f 100644 --- a/llvm/include/llvm/Object/COFF.h +++ b/llvm/include/llvm/Object/COFF.h @@ -1079,13 +1079,15 @@ public: uint64_t getImageBase() const; Error getVaPtr(uint64_t VA, uintptr_t &Res) const; - Error getRvaPtr(uint32_t Rva, uintptr_t &Res) const; + Error getRvaPtr(uint32_t Rva, uintptr_t &Res, + const char *ErrorContext = nullptr) const; /// Given an RVA base and size, returns a valid array of bytes or an error /// code if the RVA and size is not contained completely within a valid /// section. Error getRvaAndSizeAsBytes(uint32_t RVA, uint32_t Size, - ArrayRef<uint8_t> &Contents) const; + ArrayRef<uint8_t> &Contents, + const char *ErrorContext = nullptr) const; Error getHintName(uint32_t Rva, uint16_t &Hint, StringRef &Name) const; @@ -1296,6 +1298,12 @@ struct FpoData { frame_type getFP() const { return static_cast<frame_type>(Attributes >> 14); } }; +class SectionStrippedError + : public ErrorInfo<SectionStrippedError, BinaryError> { +public: + SectionStrippedError() { setErrorCode(object_error::section_stripped); } +}; + } // end namespace object } // end namespace llvm diff --git a/llvm/include/llvm/Object/COFFImportFile.h b/llvm/include/llvm/Object/COFFImportFile.h index 0da0d8fa70c9..f8f0e0343b22 100644 --- a/llvm/include/llvm/Object/COFFImportFile.h +++ b/llvm/include/llvm/Object/COFFImportFile.h @@ -18,10 +18,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Object/COFF.h" -#include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/SymbolicFile.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/raw_ostream.h" namespace llvm { diff --git a/llvm/include/llvm/Object/COFFModuleDefinition.h b/llvm/include/llvm/Object/COFFModuleDefinition.h index fb3d0952e3a3..8e14dd61472d 100644 --- a/llvm/include/llvm/Object/COFFModuleDefinition.h +++ b/llvm/include/llvm/Object/COFFModuleDefinition.h @@ -18,7 +18,7 @@ #ifndef LLVM_OBJECT_COFFMODULEDEFINITION_H #define LLVM_OBJECT_COFFMODULEDEFINITION_H -#include "llvm/Object/COFF.h" +#include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFFImportFile.h" namespace llvm { diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h new file mode 100644 index 000000000000..7aa7d8ecf4c7 --- /dev/null +++ b/llvm/include/llvm/Object/DXContainer.h @@ -0,0 +1,124 @@ +//===- DXContainer.h - DXContainer file implementation ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the DXContainerFile class, which implements the ObjectFile +// interface for DXContainer files. +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_DXCONTAINER_H +#define LLVM_OBJECT_DXCONTAINER_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/DXContainer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBufferRef.h" + +namespace llvm { +namespace object { +class DXContainer { +public: + using DXILData = std::pair<dxbc::ProgramHeader, const char *>; + +private: + DXContainer(MemoryBufferRef O); + + MemoryBufferRef Data; + dxbc::Header Header; + SmallVector<uint32_t, 4> PartOffsets; + Optional<DXILData> DXIL; + + Error parseHeader(); + Error parsePartOffsets(); + Error parseDXILHeader(uint32_t Offset); + friend class PartIterator; + +public: + // The PartIterator is a wrapper around the iterator for the PartOffsets + // member of the DXContainer. It contains a refernce to the container, and the + // current iterator value, as well as storage for a parsed part header. + class PartIterator { + const DXContainer &Container; + SmallVectorImpl<uint32_t>::const_iterator OffsetIt; + struct PartData { + dxbc::PartHeader Part; + uint32_t Offset; + StringRef Data; + } IteratorState; + + friend class DXContainer; + + PartIterator(const DXContainer &C, + SmallVectorImpl<uint32_t>::const_iterator It) + : Container(C), OffsetIt(It) { + if (OffsetIt == Container.PartOffsets.end()) + updateIteratorImpl(Container.PartOffsets.back()); + else + updateIterator(); + } + + // Updates the iterator's state data. This results in copying the part + // header into the iterator and handling any required byte swapping. This is + // called when incrementing or decrementing the iterator. + void updateIterator() { + if (OffsetIt != Container.PartOffsets.end()) + updateIteratorImpl(*OffsetIt); + } + + // Implementation for updating the iterator state based on a specified + // offest. + void updateIteratorImpl(const uint32_t Offset); + + public: + PartIterator &operator++() { + if (OffsetIt == Container.PartOffsets.end()) + return *this; + ++OffsetIt; + updateIterator(); + return *this; + } + + PartIterator operator++(int) { + PartIterator Tmp = *this; + ++(*this); + return Tmp; + } + + bool operator==(const PartIterator &RHS) const { + return OffsetIt == RHS.OffsetIt; + } + + bool operator!=(const PartIterator &RHS) const { + return OffsetIt != RHS.OffsetIt; + } + + const PartData &operator*() { return IteratorState; } + const PartData *operator->() { return &IteratorState; } + }; + + PartIterator begin() const { + return PartIterator(*this, PartOffsets.begin()); + } + + PartIterator end() const { return PartIterator(*this, PartOffsets.end()); } + + StringRef getData() const { return Data.getBuffer(); } + static Expected<DXContainer> create(MemoryBufferRef Object); + + const dxbc::Header &getHeader() const { return Header; } + + Optional<DXILData> getDXIL() const { return DXIL; } +}; + +} // namespace object +} // namespace llvm + +#endif // LLVM_OBJECT_DXCONTAINERFILE_H diff --git a/llvm/include/llvm/Object/Decompressor.h b/llvm/include/llvm/Object/Decompressor.h index cc918481b308..e04ee3c3e4c0 100644 --- a/llvm/include/llvm/Object/Decompressor.h +++ b/llvm/include/llvm/Object/Decompressor.h @@ -9,13 +9,15 @@ #ifndef LLVM_OBJECT_DECOMPRESSOR_H #define LLVM_OBJECT_DECOMPRESSOR_H -#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" namespace llvm { namespace object { +class SectionRef; + /// Decompressor helps to handle decompression of compressed sections. class Decompressor { public: diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h index 37f23c435ae1..1a59ba94098f 100644 --- a/llvm/include/llvm/Object/ELF.h +++ b/llvm/include/llvm/Object/ELF.h @@ -855,7 +855,7 @@ Expected<StringRef> ELFFile<ELFT>::getSymbolVersionByIndex( const VersionEntry &Entry = *VersionMap[VersionIndex]; // A default version (@@) is only available for defined symbols. - if (!Entry.IsVerDef || IsSymHidden.getValueOr(false)) + if (!Entry.IsVerDef || IsSymHidden.value_or(false)) IsDefault = false; else IsDefault = !(SymbolVersionIndex & llvm::ELF::VERSYM_HIDDEN); diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index e2d2784d4f23..c449a3dafc0c 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -15,7 +15,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" @@ -27,19 +26,21 @@ #include "llvm/Object/Error.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/SymbolicFile.h" -#include "llvm/Support/ARMAttributeParser.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/ELFAttributeParser.h" #include "llvm/Support/ELFAttributes.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/ScopedPrinter.h" #include <cassert> #include <cstdint> -#include <system_error> namespace llvm { + +template <typename T> class SmallVectorImpl; + namespace object { constexpr int NumElfSymbolTypes = 16; @@ -101,6 +102,12 @@ public: /// Returns a vector containing a symbol version for each dynamic symbol. /// Returns an empty vector if version sections do not exist. Expected<std::vector<VersionEntry>> readDynsymVersions() const; + + /// Returns a vector of all BB address maps in the object file. When + // `TextSectionIndex` is specified, only returns the BB address maps + // corresponding to the section with that index. + Expected<std::vector<BBAddrMap>> + readBBAddrMap(Optional<unsigned> TextSectionIndex = None) const; }; class ELFSectionRef : public SectionRef { @@ -1167,7 +1174,7 @@ uint8_t ELFObjectFile<ELFT>::getBytesInAddress() const { template <class ELFT> StringRef ELFObjectFile<ELFT>::getFileFormatName() const { - bool IsLittleEndian = ELFT::TargetEndianness == support::little; + constexpr bool IsLittleEndian = ELFT::TargetEndianness == support::little; switch (EF.getHeader().e_ident[ELF::EI_CLASS]) { case ELF::ELFCLASS32: switch (EF.getHeader().e_machine) { @@ -1202,6 +1209,8 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { return "elf32-sparc"; case ELF::EM_AMDGPU: return "elf32-amdgpu"; + case ELF::EM_LOONGARCH: + return "elf32-loongarch"; default: return "elf32-unknown"; } @@ -1229,6 +1238,8 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { return "elf64-bpf"; case ELF::EM_VE: return "elf64-ve"; + case ELF::EM_LOONGARCH: + return "elf64-loongarch"; default: return "elf64-unknown"; } @@ -1313,6 +1324,17 @@ template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const { return Triple::ve; case ELF::EM_CSKY: return Triple::csky; + + case ELF::EM_LOONGARCH: + switch (EF.getHeader().e_ident[ELF::EI_CLASS]) { + case ELF::ELFCLASS32: + return Triple::loongarch32; + case ELF::ELFCLASS64: + return Triple::loongarch64; + default: + report_fatal_error("Invalid ELFCLASS!"); + } + default: return Triple::UnknownArch; } diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index c674b80c814d..5942b6f1d0a1 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -812,8 +812,20 @@ struct BBAddrMap { : Offset(Offset), Size(Size), HasReturn(Metadata & 1), HasTailCall(Metadata & (1 << 1)), IsEHPad(Metadata & (1 << 2)), CanFallThrough(Metadata & (1 << 3)){}; + + bool operator==(const BBEntry &Other) const { + return Offset == Other.Offset && Size == Other.Size && + HasReturn == Other.HasReturn && HasTailCall == Other.HasTailCall && + IsEHPad == Other.IsEHPad && CanFallThrough == Other.CanFallThrough; + } }; std::vector<BBEntry> BBEntries; // Basic block entries for this function. + + // Equality operator for unit testing. + bool operator==(const BBAddrMap &Other) const { + return Addr == Other.Addr && std::equal(BBEntries.begin(), BBEntries.end(), + Other.BBEntries.begin()); + } }; } // end namespace object. diff --git a/llvm/include/llvm/Object/Error.h b/llvm/include/llvm/Object/Error.h index af334fc42658..8875fb6e1a20 100644 --- a/llvm/include/llvm/Object/Error.h +++ b/llvm/include/llvm/Object/Error.h @@ -34,6 +34,7 @@ enum class object_error { invalid_section_index, bitcode_section_not_found, invalid_symbol_index, + section_stripped, }; inline std::error_code make_error_code(object_error e) { diff --git a/llvm/include/llvm/Object/IRObjectFile.h b/llvm/include/llvm/Object/IRObjectFile.h index db47960237a0..6b3f2cd5671c 100644 --- a/llvm/include/llvm/Object/IRObjectFile.h +++ b/llvm/include/llvm/Object/IRObjectFile.h @@ -13,7 +13,6 @@ #ifndef LLVM_OBJECT_IROBJECTFILE_H #define LLVM_OBJECT_IROBJECTFILE_H -#include "llvm/ADT/PointerUnion.h" #include "llvm/Object/IRSymtab.h" #include "llvm/Object/ModuleSymbolTable.h" #include "llvm/Object/SymbolicFile.h" diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h index 49a0706b84be..4ec366055db6 100644 --- a/llvm/include/llvm/Object/MachO.h +++ b/llvm/include/llvm/Object/MachO.h @@ -260,6 +260,124 @@ private: }; using bind_iterator = content_iterator<MachOBindEntry>; +/// ChainedFixupTarget holds all the information about an external symbol +/// necessary to bind this binary to that symbol. These values are referenced +/// indirectly by chained fixup binds. This structure captures values from all +/// import and symbol formats. +/// +/// Be aware there are two notions of weak here: +/// WeakImport == true +/// The associated bind may be set to 0 if this symbol is missing from its +/// parent library. This is called a "weak import." +/// LibOrdinal == BIND_SPECIAL_DYLIB_WEAK_LOOKUP +/// This symbol may be coalesced with other libraries vending the same +/// symbol. E.g., C++'s "operator new". This is called a "weak bind." +struct ChainedFixupTarget { +public: + ChainedFixupTarget(int LibOrdinal, StringRef Symbol, uint64_t Addend, + bool WeakImport) + : LibOrdinal(LibOrdinal), SymbolName(Symbol), Addend(Addend), + WeakImport(WeakImport) {} + + int libOrdinal() { return LibOrdinal; } + StringRef symbolName() { return SymbolName; } + uint64_t addend() { return Addend; } + bool weakImport() { return WeakImport; } + bool weakBind() { + return LibOrdinal == MachO::BIND_SPECIAL_DYLIB_WEAK_LOOKUP; + } + +private: + int LibOrdinal; + StringRef SymbolName; + uint64_t Addend; + bool WeakImport; +}; + +/// MachOAbstractFixupEntry is an abstract class representing a fixup in a +/// MH_DYLDLINK file. Fixups generally represent rebases and binds. Binds also +/// subdivide into additional subtypes (weak, lazy, reexport). +/// +/// The two concrete subclasses of MachOAbstractFixupEntry are: +/// +/// MachORebaseBindEntry - for dyld opcode-based tables, including threaded- +/// rebase, where rebases are mixed in with other +/// bind opcodes. +/// MachOChainedFixupEntry - for pointer chains embedded in data pages. +class MachOAbstractFixupEntry { +public: + MachOAbstractFixupEntry(Error *Err, const MachOObjectFile *O); + + int32_t segmentIndex() const; + uint64_t segmentOffset() const; + uint64_t segmentAddress() const; + StringRef segmentName() const; + StringRef sectionName() const; + StringRef typeName() const; + StringRef symbolName() const; + uint32_t flags() const; + int64_t addend() const; + int ordinal() const; + + /// \return the location of this fixup as a VM Address. For the VM + /// Address this fixup is pointing to, use pointerValue(). + uint64_t address() const; + + /// \return the VM Address pointed to by this fixup. Use + /// pointerValue() to compare against other VM Addresses, such as + /// section addresses or segment vmaddrs. + uint64_t pointerValue() const { return PointerValue; } + + /// \return the raw "on-disk" representation of the fixup. For + /// Threaded rebases and Chained pointers these values are generally + /// encoded into various different pointer formats. This value is + /// exposed in API for tools that want to display and annotate the + /// raw bits. + uint64_t rawValue() const { return RawValue; } + + void moveNext(); + +protected: + Error *E; + const MachOObjectFile *O; + uint64_t SegmentOffset = 0; + int32_t SegmentIndex = -1; + StringRef SymbolName; + int32_t Ordinal = 0; + uint32_t Flags = 0; + int64_t Addend = 0; + uint64_t PointerValue = 0; + uint64_t RawValue = 0; + bool Done = false; + + void moveToFirst(); + void moveToEnd(); + + /// \return the vm address of the start of __TEXT segment. + uint64_t textAddress() const { return TextAddress; } + +private: + uint64_t TextAddress; +}; + +class MachOChainedFixupEntry : public MachOAbstractFixupEntry { +public: + enum class FixupKind { All, Bind, WeakBind, Rebase }; + + MachOChainedFixupEntry(Error *Err, const MachOObjectFile *O, bool Parse); + + bool operator==(const MachOChainedFixupEntry &) const; + + void moveNext(); + void moveToFirst(); + void moveToEnd(); + +private: + std::vector<ChainedFixupTarget> FixupTargets; + uint32_t FixupIndex = 0; +}; +using fixup_iterator = content_iterator<MachOChainedFixupEntry>; + class MachOObjectFile : public ObjectFile { public: struct LoadCommandInfo { @@ -273,6 +391,8 @@ public: create(MemoryBufferRef Object, bool IsLittleEndian, bool Is64Bits, uint32_t UniversalCputype = 0, uint32_t UniversalIndex = 0); + static bool isMachOPairedReloc(uint64_t RelocType, uint64_t Arch); + void moveSymbolNext(DataRefImpl &Symb) const override; uint64_t getNValue(DataRefImpl Sym) const; @@ -402,6 +522,9 @@ public: /// For use iterating over all bind table entries. iterator_range<bind_iterator> bindTable(Error &Err); + /// For iterating over all chained fixups. + iterator_range<fixup_iterator> fixupTable(Error &Err); + /// For use iterating over all lazy bind table entries. iterator_range<bind_iterator> lazyBindTable(Error &Err); @@ -562,7 +685,12 @@ public: ArrayRef<uint8_t> getDyldInfoBindOpcodes() const; ArrayRef<uint8_t> getDyldInfoWeakBindOpcodes() const; ArrayRef<uint8_t> getDyldInfoLazyBindOpcodes() const; + /// If the optional is None, no header was found, but the object was well-formed. + Expected<Optional<MachO::dyld_chained_fixups_header>> + getChainedFixupsHeader() const; + Expected<std::vector<ChainedFixupTarget>> getDyldChainedFixupTargets() const; ArrayRef<uint8_t> getDyldInfoExportsTrie() const; + SmallVector<uint64_t> getFunctionStarts() const; ArrayRef<uint8_t> getUuid() const; StringRef getStringTableData() const; @@ -689,6 +817,8 @@ private: const char *DataInCodeLoadCmd = nullptr; const char *LinkOptHintsLoadCmd = nullptr; const char *DyldInfoLoadCmd = nullptr; + const char *FuncStartsLoadCmd = nullptr; + const char *DyldChainedFixupsLoadCmd = nullptr; const char *UuidLoadCmd = nullptr; bool HasPageZeroSegment = false; }; diff --git a/llvm/include/llvm/Object/MachOUniversal.h b/llvm/include/llvm/Object/MachOUniversal.h index e87eb31aad4e..4fe7a68d9680 100644 --- a/llvm/include/llvm/Object/MachOUniversal.h +++ b/llvm/include/llvm/Object/MachOUniversal.h @@ -16,7 +16,6 @@ #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/MachO.h" -#include "llvm/Object/Archive.h" #include "llvm/Object/Binary.h" #include "llvm/Object/MachO.h" @@ -25,6 +24,7 @@ class StringRef; class LLVMContext; namespace object { +class Archive; class IRObjectFile; class MachOUniversalBinary : public Binary { diff --git a/llvm/include/llvm/Object/MachOUniversalWriter.h b/llvm/include/llvm/Object/MachOUniversalWriter.h index 8d095766cf48..4004f25f3fb7 100644 --- a/llvm/include/llvm/Object/MachOUniversalWriter.h +++ b/llvm/include/llvm/Object/MachOUniversalWriter.h @@ -14,15 +14,22 @@ #ifndef LLVM_OBJECT_MACHOUNIVERSALWRITER_H #define LLVM_OBJECT_MACHOUNIVERSALWRITER_H -#include "llvm/Object/Archive.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/MachO.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/Support/Error.h" +#include <cstdint> +#include <string> namespace llvm { class LLVMContext; namespace object { +class Archive; +class Binary; class IRObjectFile; +class MachOObjectFile; class Slice { const Binary *B; diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h index 950c38a599d5..8754c229bd4b 100644 --- a/llvm/include/llvm/Object/ObjectFile.h +++ b/llvm/include/llvm/Object/ObjectFile.h @@ -13,7 +13,8 @@ #ifndef LLVM_OBJECT_OBJECTFILE_H #define LLVM_OBJECT_OBJECTFILE_H -#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" @@ -24,11 +25,10 @@ #include "llvm/Object/SymbolicFile.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include <cassert> #include <cstdint> #include <memory> -#include <system_error> namespace llvm { @@ -170,11 +170,11 @@ class SymbolRef : public BasicSymbolRef { public: enum Type { ST_Unknown, // Type not specified + ST_Other, ST_Data, ST_Debug, ST_File, ST_Function, - ST_Other }; SymbolRef() = default; @@ -350,6 +350,11 @@ public: /// True if this is a relocatable object (.o/.obj). virtual bool isRelocatableObject() const = 0; + /// True if the reflection section can be stripped by the linker. + bool isReflectionSectionStrippable( + llvm::binaryformat::Swift5ReflectionSectionKind ReflectionSectionKind) + const; + /// @returns Pointer to ObjectFile subclass to handle this type of object. /// @param ObjectPath The path to the object file. ObjectPath.isObject must /// return true. diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h new file mode 100644 index 000000000000..5afc3ed295ae --- /dev/null +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -0,0 +1,169 @@ +//===--- Offloading.h - Utilities for handling offloading code -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the binary format used for budingling device metadata with +// an associated device image. The data can then be stored inside a host object +// file to create a fat binary and read by the linker. This is intended to be a +// thin wrapper around the image itself. If this format becomes sufficiently +// complex it should be moved to a standard binary format like msgpack or ELF. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_OFFLOADING_H +#define LLVM_BINARYFORMAT_OFFLOADING_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Object/Binary.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include <memory> + +namespace llvm { + +namespace object { + +/// The producer of the associated offloading image. +enum OffloadKind : uint16_t { + OFK_None = 0, + OFK_OpenMP, + OFK_Cuda, + OFK_HIP, + OFK_LAST, +}; + +/// The type of contents the offloading image contains. +enum ImageKind : uint16_t { + IMG_None = 0, + IMG_Object, + IMG_Bitcode, + IMG_Cubin, + IMG_Fatbinary, + IMG_PTX, + IMG_LAST, +}; + +/// A simple binary serialization of an offloading file. We use this format to +/// embed the offloading image into the host executable so it can be extracted +/// and used by the linker. +/// +/// Many of these could be stored in the same section by the time the linker +/// sees it so we mark this information with a header. The version is used to +/// detect ABI stability and the size is used to find other offloading entries +/// that may exist in the same section. All offsets are given as absolute byte +/// offsets from the beginning of the file. +class OffloadBinary : public Binary { +public: + using string_iterator = StringMap<StringRef>::const_iterator; + using string_iterator_range = iterator_range<string_iterator>; + + /// The current version of the binary used for backwards compatibility. + static const uint32_t Version = 1; + + /// The offloading metadata that will be serialized to a memory buffer. + struct OffloadingImage { + ImageKind TheImageKind; + OffloadKind TheOffloadKind; + uint32_t Flags; + StringMap<StringRef> StringData; + std::unique_ptr<MemoryBuffer> Image; + }; + + /// Attempt to parse the offloading binary stored in \p Data. + static Expected<std::unique_ptr<OffloadBinary>> create(MemoryBufferRef); + + /// Serialize the contents of \p File to a binary buffer to be read later. + static std::unique_ptr<MemoryBuffer> write(const OffloadingImage &); + + static uint64_t getAlignment() { return alignof(Header); } + + ImageKind getImageKind() const { return TheEntry->TheImageKind; } + OffloadKind getOffloadKind() const { return TheEntry->TheOffloadKind; } + uint32_t getVersion() const { return TheHeader->Version; } + uint32_t getFlags() const { return TheEntry->Flags; } + uint64_t getSize() const { return TheHeader->Size; } + + StringRef getTriple() const { return getString("triple"); } + StringRef getArch() const { return getString("arch"); } + StringRef getImage() const { + return StringRef(&Buffer[TheEntry->ImageOffset], TheEntry->ImageSize); + } + + // Iterator over all the key and value pairs in the binary. + string_iterator_range strings() const { + return string_iterator_range(StringData.begin(), StringData.end()); + } + + StringRef getString(StringRef Key) const { return StringData.lookup(Key); } + + static bool classof(const Binary *V) { return V->isOffloadFile(); } + + struct Header { + uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. + uint32_t Version = OffloadBinary::Version; // Version identifier. + uint64_t Size; // Size in bytes of this entire binary. + uint64_t EntryOffset; // Offset of the metadata entry in bytes. + uint64_t EntrySize; // Size of the metadata entry in bytes. + }; + + struct Entry { + ImageKind TheImageKind; // The kind of the image stored. + OffloadKind TheOffloadKind; // The producer of this image. + uint32_t Flags; // Additional flags associated with the image. + uint64_t StringOffset; // Offset in bytes to the string map. + uint64_t NumStrings; // Number of entries in the string map. + uint64_t ImageOffset; // Offset in bytes of the actual binary image. + uint64_t ImageSize; // Size in bytes of the binary image. + }; + + struct StringEntry { + uint64_t KeyOffset; + uint64_t ValueOffset; + }; + +private: + OffloadBinary(MemoryBufferRef Source, const Header *TheHeader, + const Entry *TheEntry) + : Binary(Binary::ID_Offload, Source), Buffer(Source.getBufferStart()), + TheHeader(TheHeader), TheEntry(TheEntry) { + const StringEntry *StringMapBegin = + reinterpret_cast<const StringEntry *>(&Buffer[TheEntry->StringOffset]); + for (uint64_t I = 0, E = TheEntry->NumStrings; I != E; ++I) { + StringRef Key = &Buffer[StringMapBegin[I].KeyOffset]; + StringData[Key] = &Buffer[StringMapBegin[I].ValueOffset]; + } + } + + OffloadBinary(const OffloadBinary &Other) = delete; + + /// Map from keys to offsets in the binary. + StringMap<StringRef> StringData; + /// Raw pointer to the MemoryBufferRef for convenience. + const char *Buffer; + /// Location of the header within the binary. + const Header *TheHeader; + /// Location of the metadata entries within the binary. + const Entry *TheEntry; +}; + +/// Convert a string \p Name to an image kind. +ImageKind getImageKind(StringRef Name); + +/// Convert an image kind to its string representation. +StringRef getImageKindName(ImageKind Name); + +/// Convert a string \p Name to an offload kind. +OffloadKind getOffloadKind(StringRef Name); + +/// Convert an offload kind to its string representation. +StringRef getOffloadKindName(OffloadKind Name); + +} // namespace object + +} // namespace llvm +#endif diff --git a/llvm/include/llvm/Object/RelocationResolver.h b/llvm/include/llvm/Object/RelocationResolver.h index d3b604018e89..2acdf5ed2fe1 100644 --- a/llvm/include/llvm/Object/RelocationResolver.h +++ b/llvm/include/llvm/Object/RelocationResolver.h @@ -15,22 +15,15 @@ #ifndef LLVM_OBJECT_RELOCATIONRESOLVER_H #define LLVM_OBJECT_RELOCATIONRESOLVER_H -#include "llvm/ADT/Triple.h" -#include "llvm/BinaryFormat/ELF.h" -#include "llvm/BinaryFormat/MachO.h" -#include "llvm/Object/COFF.h" -#include "llvm/Object/ELFObjectFile.h" -#include "llvm/Object/MachO.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Object/Wasm.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/ErrorHandling.h" #include <cstdint> -#include <system_error> +#include <utility> namespace llvm { namespace object { +class ObjectFile; +class RelocationRef; + using SupportsRelocation = bool (*)(uint64_t); using RelocationResolver = uint64_t (*)(uint64_t Type, uint64_t Offset, uint64_t S, uint64_t LocData, diff --git a/llvm/include/llvm/Object/SymbolicFile.h b/llvm/include/llvm/Object/SymbolicFile.h index 284302c5e042..ea51afce5d2a 100644 --- a/llvm/include/llvm/Object/SymbolicFile.h +++ b/llvm/include/llvm/Object/SymbolicFile.h @@ -13,21 +13,23 @@ #ifndef LLVM_OBJECT_SYMBOLICFILE_H #define LLVM_OBJECT_SYMBOLICFILE_H -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/Object/Binary.h" #include "llvm/Support/Error.h" #include "llvm/Support/Format.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include <cinttypes> #include <cstdint> #include <cstring> #include <iterator> #include <memory> -#include <system_error> namespace llvm { + +class LLVMContext; +class raw_ostream; + namespace object { union DataRefImpl { diff --git a/llvm/include/llvm/Object/TapiFile.h b/llvm/include/llvm/Object/TapiFile.h index ffa27fdf9654..410e58dceaf4 100644 --- a/llvm/include/llvm/Object/TapiFile.h +++ b/llvm/include/llvm/Object/TapiFile.h @@ -14,13 +14,22 @@ #define LLVM_OBJECT_TAPIFILE_H #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/iterator_range.h" +#include "llvm/Object/Binary.h" #include "llvm/Object/SymbolicFile.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/TextAPI/InterfaceFile.h" +#include "llvm/Support/MemoryBufferRef.h" +#include "llvm/TextAPI/Architecture.h" namespace llvm { + +class raw_ostream; + +namespace MachO { + +class InterfaceFile; + +} + namespace object { class TapiFile : public SymbolicFile { diff --git a/llvm/include/llvm/Object/TapiUniversal.h b/llvm/include/llvm/Object/TapiUniversal.h index ab548aa5bb2a..fff66c28c1a4 100644 --- a/llvm/include/llvm/Object/TapiUniversal.h +++ b/llvm/include/llvm/Object/TapiUniversal.h @@ -13,16 +13,18 @@ #ifndef LLVM_OBJECT_TAPIUNIVERSAL_H #define LLVM_OBJECT_TAPIUNIVERSAL_H +#include "llvm/ADT/StringRef.h" #include "llvm/Object/Binary.h" -#include "llvm/Object/TapiFile.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/InterfaceFile.h" namespace llvm { namespace object { +class TapiFile; + class TapiUniversal : public Binary { public: class ObjectForArch { diff --git a/llvm/include/llvm/Object/Wasm.h b/llvm/include/llvm/Object/Wasm.h index e4802c087b8b..abe0f6f528cc 100644 --- a/llvm/include/llvm/Object/Wasm.h +++ b/llvm/include/llvm/Object/Wasm.h @@ -287,7 +287,6 @@ private: uint32_t StartFunction = -1; bool HasLinkingSection = false; bool HasDylinkSection = false; - bool SeenCodeSection = false; bool HasMemory64 = false; wasm::WasmLinkingData LinkingData; uint32_t NumImportedGlobals = 0; diff --git a/llvm/include/llvm/Object/WindowsResource.h b/llvm/include/llvm/Object/WindowsResource.h index b8fad299c693..acda9e2659b1 100644 --- a/llvm/include/llvm/Object/WindowsResource.h +++ b/llvm/include/llvm/Object/WindowsResource.h @@ -31,7 +31,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/Binary.h" -#include "llvm/Object/COFF.h" #include "llvm/Object/Error.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryStreamReader.h" @@ -50,6 +49,7 @@ namespace object { class WindowsResource; class ResourceSectionRef; +struct coff_resource_dir_table; const size_t WIN_RES_MAGIC_SIZE = 16; const size_t WIN_RES_NULL_ENTRY_SIZE = 16; diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h index ac911e534f34..68d9afff887c 100644 --- a/llvm/include/llvm/Object/XCOFFObjectFile.h +++ b/llvm/include/llvm/Object/XCOFFObjectFile.h @@ -60,10 +60,13 @@ public: return static_cast<const T *>(this)->FlagAndTDataAlignment & AuxiHeaderFlagMask; } + uint8_t getTDataAlignment() const { return static_cast<const T *>(this)->FlagAndTDataAlignment & AuxiHeaderTDataAlignmentMask; } + + uint16_t getVersion() const { return static_cast<const T *>(this)->Version; } }; struct XCOFFAuxiliaryHeader32 : XCOFFAuxiliaryHeader<XCOFFAuxiliaryHeader32> { @@ -113,7 +116,7 @@ struct XCOFFAuxiliaryHeader32 : XCOFFAuxiliaryHeader<XCOFFAuxiliaryHeader32> { support::ubig16_t SecNumOfTBSS; }; -struct XCOFFAuxiliaryHeader64 : XCOFFAuxiliaryHeader<XCOFFAuxiliaryHeader32> { +struct XCOFFAuxiliaryHeader64 : XCOFFAuxiliaryHeader<XCOFFAuxiliaryHeader64> { support::ubig16_t AuxMagic; support::ubig16_t Version; support::ubig32_t ReservedForDebugger; @@ -448,9 +451,6 @@ private: const void *SymbolTblPtr = nullptr; XCOFFStringTable StringTable = {0, nullptr}; - const XCOFFFileHeader32 *fileHeader32() const; - const XCOFFFileHeader64 *fileHeader64() const; - const XCOFFSectionHeader32 *sectionHeaderTable32() const; const XCOFFSectionHeader64 *sectionHeaderTable64() const; template <typename T> const T *sectionHeaderTable() const; @@ -548,6 +548,8 @@ public: // Below here is the non-inherited interface. bool is64Bit() const; + Expected<StringRef> getRawData(const char *Start, uint64_t Size, + StringRef Name) const; const XCOFFAuxiliaryHeader32 *auxiliaryHeader32() const; const XCOFFAuxiliaryHeader64 *auxiliaryHeader64() const; @@ -559,6 +561,8 @@ public: XCOFFSymbolRef toSymbolRef(DataRefImpl Ref) const; // File header related interfaces. + const XCOFFFileHeader32 *fileHeader32() const; + const XCOFFFileHeader64 *fileHeader64() const; uint16_t getMagic() const; uint16_t getNumberOfSections() const; int32_t getTimeStamp() const; @@ -687,6 +691,9 @@ public: Entry32 = reinterpret_cast<const XCOFFSymbolEntry32 *>(SymEntDataRef.p); } + const XCOFFSymbolEntry32 *getSymbol32() { return Entry32; } + const XCOFFSymbolEntry64 *getSymbol64() { return Entry64; } + uint64_t getValue() const { return Entry32 ? getValue32() : getValue64(); } uint32_t getValue32() const { return Entry32->Value; } diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h new file mode 100644 index 000000000000..d1c0cd912d97 --- /dev/null +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -0,0 +1,101 @@ +//===- DXContainerYAML.h - DXContainer YAMLIO implementation ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares classes for handling the YAML representation +/// of DXContainer. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECTYAML_DXCONTAINERYAML_H +#define LLVM_OBJECTYAML_DXCONTAINERYAML_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/YAMLTraits.h" +#include <cstdint> +#include <string> +#include <vector> + +namespace llvm { +namespace DXContainerYAML { + +struct VersionTuple { + uint16_t Major; + uint16_t Minor; +}; + +// The optional header fields are required in the binary and will be populated +// when reading from binary, but can be omitted in the YAML text because the +// emitter can calculate them. +struct FileHeader { + std::vector<llvm::yaml::Hex8> Hash; + VersionTuple Version; + Optional<uint32_t> FileSize; + uint32_t PartCount; + Optional<std::vector<uint32_t>> PartOffsets; +}; + +struct DXILProgram { + uint8_t MajorVersion; + uint8_t MinorVersion; + uint16_t ShaderKind; + Optional<uint32_t> Size; + uint16_t DXILMajorVersion; + uint16_t DXILMinorVersion; + Optional<uint32_t> DXILOffset; + Optional<uint32_t> DXILSize; + Optional<std::vector<llvm::yaml::Hex8>> DXIL; +}; + +struct Part { + std::string Name; + uint32_t Size; + Optional<DXILProgram> Program; +}; + +struct Object { + FileHeader Header; + std::vector<Part> Parts; +}; + +} // namespace DXContainerYAML +} // namespace llvm + +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::Part) +namespace llvm { + +class raw_ostream; + +namespace yaml { + +template <> struct MappingTraits<DXContainerYAML::VersionTuple> { + static void mapping(IO &IO, DXContainerYAML::VersionTuple &Version); +}; + +template <> struct MappingTraits<DXContainerYAML::FileHeader> { + static void mapping(IO &IO, DXContainerYAML::FileHeader &Header); +}; + +template <> struct MappingTraits<DXContainerYAML::DXILProgram> { + static void mapping(IO &IO, DXContainerYAML::DXILProgram &Program); +}; + +template <> struct MappingTraits<DXContainerYAML::Part> { + static void mapping(IO &IO, DXContainerYAML::Part &Version); +}; + +template <> struct MappingTraits<DXContainerYAML::Object> { + static void mapping(IO &IO, DXContainerYAML::Object &Obj); +}; + +} // namespace yaml + +} // namespace llvm + +#endif // LLVM_OBJECTYAML_DXCONTAINERYAML_H diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h index 92a9f78ce7bf..ddd5dd9cf3c9 100644 --- a/llvm/include/llvm/ObjectYAML/ELFYAML.h +++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h @@ -161,6 +161,8 @@ struct BBAddrMapEntry { llvm::yaml::Hex64 Size; llvm::yaml::Hex64 Metadata; }; + uint8_t Version; + llvm::yaml::Hex8 Feature; llvm::yaml::Hex64 Address; Optional<uint64_t> NumBlocks; Optional<std::vector<BBEntry>> BBEntries; @@ -317,7 +319,7 @@ struct BBAddrMapSection : Section { BBAddrMapSection() : Section(ChunkKind::BBAddrMap) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -331,7 +333,7 @@ struct StackSizesSection : Section { StackSizesSection() : Section(ChunkKind::StackSizes) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -349,7 +351,7 @@ struct DynamicSection : Section { DynamicSection() : Section(ChunkKind::Dynamic) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Dynamic; } @@ -380,7 +382,7 @@ struct NoteSection : Section { NoteSection() : Section(ChunkKind::Note) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Notes", Notes.hasValue()}}; + return {{"Notes", Notes.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Note; } @@ -391,7 +393,7 @@ struct HashSection : Section { Optional<std::vector<uint32_t>> Chain; std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Bucket", Bucket.hasValue()}, {"Chain", Chain.hasValue()}}; + return {{"Bucket", Bucket.has_value()}, {"Chain", Chain.has_value()}}; }; // The following members are used to override section fields. @@ -433,10 +435,10 @@ struct GnuHashSection : Section { GnuHashSection() : Section(ChunkKind::GnuHash) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Header", Header.hasValue()}, - {"BloomFilter", BloomFilter.hasValue()}, - {"HashBuckets", HashBuckets.hasValue()}, - {"HashValues", HashValues.hasValue()}}; + return {{"Header", Header.has_value()}, + {"BloomFilter", BloomFilter.has_value()}, + {"HashBuckets", HashBuckets.has_value()}, + {"HashValues", HashValues.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::GnuHash; } @@ -462,7 +464,7 @@ struct VerneedSection : Section { VerneedSection() : Section(ChunkKind::Verneed) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Dependencies", VerneedV.hasValue()}}; + return {{"Dependencies", VerneedV.has_value()}}; }; static bool classof(const Chunk *S) { @@ -476,7 +478,7 @@ struct AddrsigSection : Section { AddrsigSection() : Section(ChunkKind::Addrsig) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Symbols", Symbols.hasValue()}}; + return {{"Symbols", Symbols.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Addrsig; } @@ -493,7 +495,7 @@ struct LinkerOptionsSection : Section { LinkerOptionsSection() : Section(ChunkKind::LinkerOptions) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Options", Options.hasValue()}}; + return {{"Options", Options.has_value()}}; }; static bool classof(const Chunk *S) { @@ -507,7 +509,7 @@ struct DependentLibrariesSection : Section { DependentLibrariesSection() : Section(ChunkKind::DependentLibraries) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Libraries", Libs.hasValue()}}; + return {{"Libraries", Libs.has_value()}}; }; static bool classof(const Chunk *S) { @@ -527,7 +529,7 @@ struct CallGraphProfileSection : Section { CallGraphProfileSection() : Section(ChunkKind::CallGraphProfile) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -541,7 +543,7 @@ struct SymverSection : Section { SymverSection() : Section(ChunkKind::Symver) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Symver; } @@ -562,7 +564,7 @@ struct VerdefSection : Section { VerdefSection() : Section(ChunkKind::Verdef) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Verdef; } @@ -577,7 +579,7 @@ struct GroupSection : Section { GroupSection() : Section(ChunkKind::Group) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Members", Members.hasValue()}}; + return {{"Members", Members.has_value()}}; }; static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Group; } @@ -597,7 +599,7 @@ struct RelocationSection : Section { RelocationSection() : Section(ChunkKind::Relocation) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Relocations", Relocations.hasValue()}}; + return {{"Relocations", Relocations.has_value()}}; }; static bool classof(const Chunk *S) { @@ -611,7 +613,7 @@ struct RelrSection : Section { RelrSection() : Section(ChunkKind::Relr) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -625,7 +627,7 @@ struct SymtabShndxSection : Section { SymtabShndxSection() : Section(ChunkKind::SymtabShndxSection) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -644,7 +646,7 @@ struct ARMIndexTableSection : Section { ARMIndexTableSection() : Section(ChunkKind::ARMIndexTable) {} std::vector<std::pair<StringRef, bool>> getEntries() const override { - return {{"Entries", Entries.hasValue()}}; + return {{"Entries", Entries.has_value()}}; }; static bool classof(const Chunk *S) { @@ -720,6 +722,7 @@ struct Object { llvm_unreachable("the section header table chunk must always be present"); } + ELF_ELFOSABI getOSAbi() const; unsigned getMachine() const; }; diff --git a/llvm/include/llvm/ObjectYAML/MachOYAML.h b/llvm/include/llvm/ObjectYAML/MachOYAML.h index 38a7de3d6131..095377c1b824 100644 --- a/llvm/include/llvm/ObjectYAML/MachOYAML.h +++ b/llvm/include/llvm/ObjectYAML/MachOYAML.h @@ -122,6 +122,7 @@ struct LinkEditData { std::vector<NListEntry> NameList; std::vector<StringRef> StringTable; std::vector<yaml::Hex32> IndirectSymbols; + std::vector<yaml::Hex64> FunctionStarts; bool isEmpty() const; }; diff --git a/llvm/include/llvm/ObjectYAML/ObjectYAML.h b/llvm/include/llvm/ObjectYAML/ObjectYAML.h index 312777aadd4c..b63607e6796b 100644 --- a/llvm/include/llvm/ObjectYAML/ObjectYAML.h +++ b/llvm/include/llvm/ObjectYAML/ObjectYAML.h @@ -11,9 +11,11 @@ #include "llvm/ObjectYAML/ArchiveYAML.h" #include "llvm/ObjectYAML/COFFYAML.h" +#include "llvm/ObjectYAML/DXContainerYAML.h" #include "llvm/ObjectYAML/ELFYAML.h" #include "llvm/ObjectYAML/MachOYAML.h" #include "llvm/ObjectYAML/MinidumpYAML.h" +#include "llvm/ObjectYAML/OffloadYAML.h" #include "llvm/ObjectYAML/WasmYAML.h" #include "llvm/ObjectYAML/XCOFFYAML.h" #include "llvm/Support/YAMLTraits.h" @@ -31,8 +33,10 @@ struct YamlObjectFile { std::unique_ptr<MachOYAML::Object> MachO; std::unique_ptr<MachOYAML::UniversalBinary> FatMachO; std::unique_ptr<MinidumpYAML::Object> Minidump; + std::unique_ptr<OffloadYAML::Binary> Offload; std::unique_ptr<WasmYAML::Object> Wasm; std::unique_ptr<XCOFFYAML::Object> Xcoff; + std::unique_ptr<DXContainerYAML::Object> DXContainer; }; template <> struct MappingTraits<YamlObjectFile> { diff --git a/llvm/include/llvm/ObjectYAML/OffloadYAML.h b/llvm/include/llvm/ObjectYAML/OffloadYAML.h new file mode 100644 index 000000000000..a4fdbce0b320 --- /dev/null +++ b/llvm/include/llvm/ObjectYAML/OffloadYAML.h @@ -0,0 +1,79 @@ +//===- OffloadYAML.h - Offload Binary YAMLIO implementation -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares classes for handling the YAML representation of +/// offloading binaries. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECTYAML_OFFLOADYAML_H +#define LLVM_OBJECTYAML_OFFLOADYAML_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/Object/OffloadBinary.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/YAMLTraits.h" + +namespace llvm { +namespace OffloadYAML { + +struct Binary { + struct StringEntry { + StringRef Key; + StringRef Value; + }; + + struct Member { + Optional<object::ImageKind> ImageKind; + Optional<object::OffloadKind> OffloadKind; + Optional<uint32_t> Flags; + Optional<std::vector<StringEntry>> StringEntries; + Optional<yaml::BinaryRef> Content; + }; + + Optional<uint32_t> Version; + Optional<uint64_t> Size; + Optional<uint64_t> EntryOffset; + Optional<uint64_t> EntrySize; + std::vector<Member> Members; +}; + +} // end namespace OffloadYAML +} // end namespace llvm + +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::OffloadYAML::Binary::Member) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::OffloadYAML::Binary::StringEntry) + +namespace llvm { +namespace yaml { + +template <> struct ScalarEnumerationTraits<object::ImageKind> { + static void enumeration(IO &IO, object::ImageKind &Value); +}; + +template <> struct ScalarEnumerationTraits<object::OffloadKind> { + static void enumeration(IO &IO, object::OffloadKind &Value); +}; + +template <> struct MappingTraits<OffloadYAML::Binary> { + static void mapping(IO &IO, OffloadYAML::Binary &O); +}; + +template <> struct MappingTraits<OffloadYAML::Binary::StringEntry> { + static void mapping(IO &IO, OffloadYAML::Binary::StringEntry &M); +}; + +template <> struct MappingTraits<OffloadYAML::Binary::Member> { + static void mapping(IO &IO, OffloadYAML::Binary::Member &M); +}; + +} // end namespace yaml +} // end namespace llvm + +#endif // LLVM_OBJECTYAML_ARCHIVEYAML_H diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h index e3a1ba0d58a6..0f6c4f06665f 100644 --- a/llvm/include/llvm/ObjectYAML/WasmYAML.h +++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h @@ -62,11 +62,20 @@ struct Export { uint32_t Index; }; +struct InitExpr { + InitExpr() {} + bool Extended; + union { + wasm::WasmInitExprMVP Inst; + yaml::BinaryRef Body; + }; +}; + struct ElemSegment { uint32_t Flags; uint32_t TableNumber; ValueType ElemKind; - wasm::WasmInitExpr Offset; + InitExpr Offset; std::vector<uint32_t> Functions; }; @@ -74,19 +83,20 @@ struct Global { uint32_t Index; ValueType Type; bool Mutable; - wasm::WasmInitExpr InitExpr; + InitExpr Init; }; struct Import { + Import() {} StringRef Module; StringRef Field; ExportKind Kind; union { uint32_t SigIndex; - Global GlobalImport; Table TableImport; Limits Memory; uint32_t TagIndex; + Global GlobalImport; }; }; @@ -114,7 +124,7 @@ struct DataSegment { uint32_t SectionOffset; uint32_t InitFlags; uint32_t MemoryIndex; - wasm::WasmInitExpr Offset; + InitExpr Offset; yaml::BinaryRef Content; }; @@ -526,8 +536,8 @@ template <> struct MappingTraits<WasmYAML::LocalDecl> { static void mapping(IO &IO, WasmYAML::LocalDecl &LocalDecl); }; -template <> struct MappingTraits<wasm::WasmInitExpr> { - static void mapping(IO &IO, wasm::WasmInitExpr &Expr); +template <> struct MappingTraits<WasmYAML::InitExpr> { + static void mapping(IO &IO, WasmYAML::InitExpr &Expr); }; template <> struct MappingTraits<WasmYAML::DataSegment> { diff --git a/llvm/include/llvm/ObjectYAML/yaml2obj.h b/llvm/include/llvm/ObjectYAML/yaml2obj.h index 468f673fd451..000da077bb18 100644 --- a/llvm/include/llvm/ObjectYAML/yaml2obj.h +++ b/llvm/include/llvm/ObjectYAML/yaml2obj.h @@ -36,6 +36,10 @@ namespace MinidumpYAML { struct Object; } +namespace OffloadYAML { +struct Binary; +} + namespace WasmYAML { struct Object; } @@ -48,6 +52,10 @@ namespace ArchYAML { struct Archive; } +namespace DXContainerYAML { +struct Object; +} // namespace DXContainerYAML + namespace yaml { class Input; struct YamlObjectFile; @@ -61,8 +69,11 @@ bool yaml2elf(ELFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH, bool yaml2macho(YamlObjectFile &Doc, raw_ostream &Out, ErrorHandler EH); bool yaml2minidump(MinidumpYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); +bool yaml2offload(OffloadYAML::Binary &Doc, raw_ostream &Out, ErrorHandler EH); bool yaml2wasm(WasmYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); bool yaml2xcoff(XCOFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH); +bool yaml2dxcontainer(DXContainerYAML::Object &Doc, raw_ostream &Out, + ErrorHandler EH); bool convertYAML(Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler, unsigned DocNum = 1, uint64_t MaxSize = UINT64_MAX); diff --git a/llvm/include/llvm/Option/ArgList.h b/llvm/include/llvm/Option/ArgList.h index 74897de52a93..6a07e1c657dc 100644 --- a/llvm/include/llvm/Option/ArgList.h +++ b/llvm/include/llvm/Option/ArgList.h @@ -298,14 +298,24 @@ public: /// true if the option is present, false if the negation is present, and /// \p Default if neither option is given. If both the option and its /// negation are present, the last one wins. - bool hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default=true) const; + bool hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const; /// hasFlag - Given an option \p Pos, an alias \p PosAlias and its negative /// form \p Neg, return true if the option or its alias is present, false if /// the negation is present, and \p Default if none of the options are /// given. If multiple options are present, the last one wins. bool hasFlag(OptSpecifier Pos, OptSpecifier PosAlias, OptSpecifier Neg, - bool Default = true) const; + bool Default) const; + + /// Given an option Pos and its negative form Neg, render the option if Pos is + /// present. + void addOptInFlag(ArgStringList &Output, OptSpecifier Pos, + OptSpecifier Neg) const; + /// Render the option if Neg is present. + void addOptOutFlag(ArgStringList &Output, OptSpecifier Pos, + OptSpecifier Neg) const { + addOptInFlag(Output, Neg, Pos); + } /// Render only the last argument match \p Id0, if present. template<typename ...OptSpecifiers> diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h index 8aa9ba90a9ca..6445e16ab68f 100644 --- a/llvm/include/llvm/Pass.h +++ b/llvm/include/llvm/Pass.h @@ -228,6 +228,16 @@ public: template <typename AnalysisType> AnalysisType &getAnalysisID(AnalysisID PI, Function &F, bool *Changed = nullptr); + +#ifdef EXPENSIVE_CHECKS + /// Hash a module in order to detect when a module (or more specific) pass has + /// modified it. + uint64_t structuralHash(Module &M) const; + + /// Hash a function in order to detect when a function (or more specific) pass + /// has modified it. + virtual uint64_t structuralHash(Function &F) const; +#endif }; //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 66b0b149fa25..0cbbdf7f3ce8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -215,8 +215,9 @@ public: /// only intended for use when attempting to optimize code. If frontends /// require some transformations for semantic reasons, they should explicitly /// build them. - ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, - bool LTOPreLink = false); + ModulePassManager + buildModuleOptimizationPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase LTOPhase); /// Build a per-module default optimization pipeline. /// @@ -470,6 +471,15 @@ public: /// Register a callback for a default optimizer pipeline extension point /// + /// This extension point allows adding optimizations before the function + /// optimization pipeline. + void registerOptimizerEarlyEPCallback( + const std::function<void(ModulePassManager &, OptimizationLevel)> &C) { + OptimizerEarlyEPCallbacks.push_back(C); + } + + /// Register a callback for a default optimizer pipeline extension point + /// /// This extension point allows adding optimizations at the very end of the /// function optimization pipeline. void registerOptimizerLastEPCallback( @@ -477,6 +487,24 @@ public: OptimizerLastEPCallbacks.push_back(C); } + /// Register a callback for a default optimizer pipeline extension point + /// + /// This extension point allows adding optimizations at the start of the full + /// LTO pipeline. + void registerFullLinkTimeOptimizationEarlyEPCallback( + const std::function<void(ModulePassManager &, OptimizationLevel)> &C) { + FullLinkTimeOptimizationEarlyEPCallbacks.push_back(C); + } + + /// Register a callback for a default optimizer pipeline extension point + /// + /// This extension point allows adding optimizations at the end of the full + /// LTO pipeline. + void registerFullLinkTimeOptimizationLastEPCallback( + const std::function<void(ModulePassManager &, OptimizationLevel)> &C) { + FullLinkTimeOptimizationLastEPCallbacks.push_back(C); + } + /// Register a callback for parsing an AliasAnalysis Name to populate /// the given AAManager \p AA void registerParseAACallback( @@ -582,7 +610,8 @@ private: void addPGOInstrPasses(ModulePassManager &MPM, OptimizationLevel Level, bool RunProfileGen, bool IsCS, std::string ProfileFile, - std::string ProfileRemappingFile); + std::string ProfileRemappingFile, + ThinOrFullLTOPhase LTOPhase); void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); // Extension Point callbacks @@ -598,9 +627,15 @@ private: CGSCCOptimizerLateEPCallbacks; SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2> VectorizerStartEPCallbacks; + // Module callbacks + SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> + OptimizerEarlyEPCallbacks; SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> OptimizerLastEPCallbacks; - // Module callbacks + SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> + FullLinkTimeOptimizationEarlyEPCallbacks; + SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> + FullLinkTimeOptimizationLastEPCallbacks; SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> PipelineStartEPCallbacks; SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2> diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h index 561cd54fa998..32ecc9ec5fb0 100644 --- a/llvm/include/llvm/Passes/StandardInstrumentations.h +++ b/llvm/include/llvm/Passes/StandardInstrumentations.h @@ -187,17 +187,6 @@ protected: // Register required callbacks. void registerRequiredCallbacks(PassInstrumentationCallbacks &PIC); - // Return true when this is a defined function for which printing - // of changes is desired. - bool isInterestingFunction(const Function &F); - - // Return true when this is a pass for which printing of changes is desired. - bool isInterestingPass(StringRef PassID); - - // Return true when this is a pass on IR for which printing - // of changes is desired. - bool isInteresting(Any IR, StringRef PassID); - // Called on the first IR processed. virtual void handleInitialIR(Any IR) = 0; // Called before and after a pass to get the representation of the IR. @@ -491,6 +480,25 @@ protected: std::unique_ptr<raw_fd_ostream> HTML; }; +// Print IR on crash. +class PrintCrashIRInstrumentation { +public: + PrintCrashIRInstrumentation() + : SavedIR("*** Dump of IR Before Last Pass Unknown ***") {} + ~PrintCrashIRInstrumentation(); + void registerCallbacks(PassInstrumentationCallbacks &PIC); + void reportCrashIR(); + +protected: + std::string SavedIR; + +private: + // The crash reporter that will report on a crash. + static PrintCrashIRInstrumentation *CrashReporter; + // Crash handler registered when print-on-crash is specified. + static void SignalHandler(void *); +}; + /// This class provides an interface to register all the standard pass /// instrumentations and manages their state (if any). class StandardInstrumentations { @@ -504,6 +512,7 @@ class StandardInstrumentations { PseudoProbeVerifier PseudoProbeVerification; InLineChangePrinter PrintChangedDiff; DotCfgChangeReporter WebsiteChangeReporter; + PrintCrashIRInstrumentation PrintCrashIR; VerifyInstrumentation Verify; bool VerifyEach; diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index e1f45019b1a9..e35751512245 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -195,11 +195,11 @@ public: ArrayRef<CounterExpression> getExpressions() const { return Expressions; } /// Return a counter that represents the expression that adds LHS and RHS. - Counter add(Counter LHS, Counter RHS); + Counter add(Counter LHS, Counter RHS, bool Simplify = true); /// Return a counter that represents the expression that subtracts RHS from /// LHS. - Counter subtract(Counter LHS, Counter RHS); + Counter subtract(Counter LHS, Counter RHS, bool Simplify = true); }; using LineColPair = std::pair<unsigned, unsigned>; diff --git a/llvm/include/llvm/ProfileData/GCOV.h b/llvm/include/llvm/ProfileData/GCOV.h index ef6515d39144..fe56f84f28b6 100644 --- a/llvm/include/llvm/ProfileData/GCOV.h +++ b/llvm/include/llvm/ProfileData/GCOV.h @@ -14,9 +14,7 @@ #ifndef LLVM_PROFILEDATA_GCOV_H #define LLVM_PROFILEDATA_GCOV_H -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" @@ -26,10 +24,8 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> -#include <cassert> #include <cstddef> #include <cstdint> -#include <limits> #include <map> #include <memory> #include <string> diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index a416eb28906e..401d278cbd06 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -281,13 +281,21 @@ bool needsComdatForCounter(const Function &F, const Module &M); /// An enum describing the attributes of an instrumented profile. enum class InstrProfKind { Unknown = 0x0, - FE = 0x1, // A frontend clang profile, incompatible with other attrs. - IR = 0x2, // An IR-level profile (default when -fprofile-generate is used). - BB = 0x4, // A profile with entry basic block instrumentation. - CS = 0x8, // A context sensitive IR-level profile. - SingleByteCoverage = 0x10, // Use single byte probes for coverage. - FunctionEntryOnly = 0x20, // Only instrument the function entry basic block. - LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FunctionEntryOnly) + // A frontend clang profile, incompatible with other attrs. + FrontendInstrumentation = 0x1, + // An IR-level profile (default when -fprofile-generate is used). + IRInstrumentation = 0x2, + // A profile with entry basic block instrumentation. + FunctionEntryInstrumentation = 0x4, + // A context sensitive IR-level profile. + ContextSensitive = 0x8, + // Use single byte probes for coverage. + SingleByteCoverage = 0x10, + // Only instrument the function entry basic block. + FunctionEntryOnly = 0x20, + // A memory profile collected using -fprofile=memory. + MemProf = 0x40, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/MemProf) }; const std::error_category &instrprof_category(); @@ -1011,7 +1019,9 @@ enum ProfVersion { Version6 = 6, // An additional counter is added around logical operators. Version7 = 7, - // The current version is 7. + // An additional (optional) memory profile type is added. + Version8 = 8, + // The current version is 8. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1028,6 +1038,21 @@ struct Header { uint64_t Unused; // Becomes unused since version 4 uint64_t HashType; uint64_t HashOffset; + uint64_t MemProfOffset; + // New fields should only be added at the end to ensure that the size + // computation is correct. The methods below need to be updated to ensure that + // the new field is read correctly. + + // Reads a header struct from the buffer. + static Expected<Header> readFromBuffer(const unsigned char *Buffer); + + // Returns the size of the header in bytes for all valid fields based on the + // version. I.e a older version header will return a smaller size. + size_t size() const; + + // Returns the format version in little endian. The header retains the version + // in native endian of the compiler runtime. + uint64_t formatVersion() const; }; // Profile summary data recorded in the profile data file in indexed diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index 3d0076fd9035..79995c813266 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -13,16 +13,17 @@ #define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H #include "llvm/ADT/DenseSet.h" -#include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/ObjectFile.h" #include "llvm/ProfileData/InstrProf.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include <vector> namespace llvm { +class DWARFContext; +class DWARFDie; +namespace object { +class ObjectFile; +} /// InstrProfCorrelator - A base class used to create raw instrumentation data /// to their functions. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 62054a6a3df5..282620d8b5dc 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -650,7 +650,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 7 +#define INSTR_PROF_INDEX_VERSION 8 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 5 @@ -662,6 +662,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. * The 61st bit indicates function entry instrumentation only. + * The 62nd bit indicates whether memory profile information is present. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) @@ -671,6 +672,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) #define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) #define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) +#define VARIANT_MASK_MEMPROF (0x1ULL << 62) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index e9dd19a69792..3a25de05bbf1 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -19,6 +19,7 @@ #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfCorrelator.h" +#include "llvm/ProfileData/MemProf.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" @@ -39,25 +40,36 @@ namespace llvm { class InstrProfReader; /// A file format agnostic iterator over profiling data. +template <class record_type = NamedInstrProfRecord, + class reader_type = InstrProfReader> class InstrProfIterator { public: using iterator_category = std::input_iterator_tag; - using value_type = NamedInstrProfRecord; + using value_type = record_type; using difference_type = std::ptrdiff_t; using pointer = value_type *; using reference = value_type &; private: - InstrProfReader *Reader = nullptr; + reader_type *Reader = nullptr; value_type Record; - void Increment(); + void increment() { + if (Error E = Reader->readNextRecord(Record)) { + // Handle errors in the reader. + InstrProfError::take(std::move(E)); + *this = InstrProfIterator(); + } + } public: InstrProfIterator() = default; - InstrProfIterator(InstrProfReader *Reader) : Reader(Reader) { Increment(); } + InstrProfIterator(reader_type *Reader) : Reader(Reader) { increment(); } - InstrProfIterator &operator++() { Increment(); return *this; } + InstrProfIterator &operator++() { + increment(); + return *this; + } bool operator==(const InstrProfIterator &RHS) const { return Reader == RHS.Reader; } @@ -88,8 +100,8 @@ public: virtual Error printBinaryIds(raw_ostream &OS) { return success(); }; /// Iterator over profile data. - InstrProfIterator begin() { return InstrProfIterator(this); } - InstrProfIterator end() { return InstrProfIterator(); } + InstrProfIterator<> begin() { return InstrProfIterator<>(this); } + InstrProfIterator<> end() { return InstrProfIterator<>(); } virtual bool isIRLevelProfile() const = 0; @@ -201,15 +213,16 @@ public: static bool hasFormat(const MemoryBuffer &Buffer); bool isIRLevelProfile() const override { - return static_cast<bool>(ProfileKind & InstrProfKind::IR); + return static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation); } bool hasCSIRLevelProfile() const override { - return static_cast<bool>(ProfileKind & InstrProfKind::CS); + return static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive); } bool instrEntryBBEnabled() const override { - return static_cast<bool>(ProfileKind & InstrProfKind::BB); + return static_cast<bool>(ProfileKind & + InstrProfKind::FunctionEntryInstrumentation); } bool hasSingleByteCoverage() const override { @@ -460,6 +473,11 @@ struct InstrProfReaderIndexBase { using OnDiskHashTableImplV3 = OnDiskIterableChainedHashTable<InstrProfLookupTrait>; +using MemProfRecordHashTable = + OnDiskIterableChainedHashTable<memprof::RecordLookupTrait>; +using MemProfFrameHashTable = + OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>; + template <typename HashTableImpl> class InstrProfReaderItaniumRemapper; @@ -545,6 +563,13 @@ private: std::unique_ptr<ProfileSummary> Summary; /// Context sensitive profile summary data. std::unique_ptr<ProfileSummary> CS_Summary; + /// MemProf profile schema (if available). + memprof::MemProfSchema Schema; + /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName). + std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable; + /// MemProf frame profile data on-disk indexed via frame id. + std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable; + // Index to the current record in the record array. unsigned RecordIndex; @@ -598,6 +623,10 @@ public: Expected<InstrProfRecord> getInstrProfRecord(StringRef FuncName, uint64_t FuncHash); + /// Return the memprof record for the function identified by + /// llvm::md5(Name). + Expected<memprof::MemProfRecord> getMemProfRecord(uint64_t FuncNameHash); + /// Fill Counts with the profile data for the given function name. Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector<uint64_t> &Counts); diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index af1e46cf4fc2..29e07961a2f4 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -15,11 +15,13 @@ #define LLVM_PROFILEDATA_INSTRPROFWRITER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/MemProf.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" #include <cstdint> #include <memory> @@ -28,6 +30,7 @@ namespace llvm { /// Writer for instrumentation based profile data. class InstrProfRecordWriterTrait; class ProfOStream; +class MemoryBuffer; class raw_fd_ostream; class InstrProfWriter { @@ -37,6 +40,16 @@ public: private: bool Sparse; StringMap<ProfilingData> FunctionData; + + // A map to hold memprof data per function. The lower 64 bits obtained from + // the md5 hash of the function name is used to index into the map. + llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> + MemProfRecordData; + // A map to hold frame id to frame mappings. The mappings are used to + // convert IndexedMemProfRecord to MemProfRecords with frame information + // inline. + llvm::MapVector<memprof::FrameId, memprof::Frame> MemProfFrameData; + // An enum describing the attributes of the profile. InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. @@ -57,6 +70,15 @@ public: addRecord(std::move(I), 1, Warn); } + /// Add a memprof record for a function identified by its \p Id. + void addMemProfRecord(const GlobalValue::GUID Id, + const memprof::IndexedMemProfRecord &Record); + + /// Add a memprof frame identified by the hash of the contents of the frame in + /// \p FrameId. + bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, + function_ref<void(Error)> Warn); + /// Merge existing function counts from the given writer. void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref<void(Error)> Warn); @@ -97,11 +119,13 @@ public: // Check if the profiles are in-compatible. Clang frontend profiles can't be // merged with other profile types. - if (static_cast<bool>((ProfileKind & InstrProfKind::FE) ^ - (Other & InstrProfKind::FE))) { + if (static_cast<bool>( + (ProfileKind & InstrProfKind::FrontendInstrumentation) ^ + (Other & InstrProfKind::FrontendInstrumentation))) { return make_error<InstrProfError>(instrprof_error::unsupported_version); } - if (testIncompatible(InstrProfKind::FunctionEntryOnly, InstrProfKind::BB)) { + if (testIncompatible(InstrProfKind::FunctionEntryOnly, + InstrProfKind::FunctionEntryInstrumentation)) { return make_error<InstrProfError>( instrprof_error::unsupported_version, "cannot merge FunctionEntryOnly profiles and BB profiles together"); @@ -112,6 +136,8 @@ public: return Error::success(); } + InstrProfKind getProfileKind() const { return ProfileKind; } + // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); void setOutputSparse(bool Sparse); diff --git a/llvm/include/llvm/ProfileData/MIBEntryDef.inc b/llvm/include/llvm/ProfileData/MIBEntryDef.inc new file mode 100644 index 000000000000..f5c6f0e4924b --- /dev/null +++ b/llvm/include/llvm/ProfileData/MIBEntryDef.inc @@ -0,0 +1,47 @@ +/*===-- MemEntryDef.inc - MemProf profiling runtime macros -*- C++ -*-======== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This file defines the macros for memprof profiling data structures. + * Eg. usage to define the memprof meminfoblock struct: + * + * struct MemInfoBlock { + * #define MIBEntryDef(NameTag, Name, Type) Type Name; + * #include MIBEntryDef.inc + * #undef MIBEntryDef + * }; + * + * This file has two identical copies. The primary copy lives in LLVM and + * the other one sits in compiler-rt/include/profile directory. To make changes + * in this file, first modify the primary copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * +\*===----------------------------------------------------------------------===*/ +#ifndef MIBEntryDef +#define MIBEntryDef(NameTag, Name, Type) +#endif + +MIBEntryDef(AllocCount = 1, AllocCount, uint32_t) +MIBEntryDef(TotalAccessCount = 2, TotalAccessCount, uint64_t) +MIBEntryDef(MinAccessCount = 3, MinAccessCount, uint64_t) +MIBEntryDef(MaxAccessCount = 4, MaxAccessCount, uint64_t) +MIBEntryDef(TotalSize = 5, TotalSize, uint64_t) +MIBEntryDef(MinSize = 6, MinSize, uint32_t) +MIBEntryDef(MaxSize = 7, MaxSize, uint32_t) +MIBEntryDef(AllocTimestamp = 8, AllocTimestamp, uint32_t) +MIBEntryDef(DeallocTimestamp = 9, DeallocTimestamp, uint32_t) +MIBEntryDef(TotalLifetime = 10, TotalLifetime, uint64_t) +MIBEntryDef(MinLifetime = 11, MinLifetime, uint32_t) +MIBEntryDef(MaxLifetime = 12, MaxLifetime, uint32_t) +MIBEntryDef(AllocCpuId = 13, AllocCpuId, uint32_t) +MIBEntryDef(DeallocCpuId = 14, DeallocCpuId, uint32_t) +MIBEntryDef(NumMigratedCpu = 15, NumMigratedCpu, uint32_t) +MIBEntryDef(NumLifetimeOverlaps = 16, NumLifetimeOverlaps, uint32_t) +MIBEntryDef(NumSameAllocCpu = 17, NumSameAllocCpu, uint32_t) +MIBEntryDef(NumSameDeallocCpu = 18, NumSameDeallocCpu, uint32_t) +MIBEntryDef(DataTypeId = 19, DataTypeId, uint64_t) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h new file mode 100644 index 000000000000..bcee3b25bf87 --- /dev/null +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -0,0 +1,613 @@ +#ifndef LLVM_PROFILEDATA_MEMPROF_H_ +#define LLVM_PROFILEDATA_MEMPROF_H_ + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/ProfileData/MemProfData.inc" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/raw_ostream.h" + +#include <cstdint> + +namespace llvm { +namespace memprof { + +enum class Meta : uint64_t { + Start = 0, +#define MIBEntryDef(NameTag, Name, Type) NameTag, +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + Size +}; + +using MemProfSchema = llvm::SmallVector<Meta, static_cast<int>(Meta::Size)>; + +// Holds the actual MemInfoBlock data with all fields. Contents may be read or +// written partially by providing an appropriate schema to the serialize and +// deserialize methods. +struct PortableMemInfoBlock { + PortableMemInfoBlock() = default; + explicit PortableMemInfoBlock(const MemInfoBlock &Block) { +#define MIBEntryDef(NameTag, Name, Type) Name = Block.Name; +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + } + + PortableMemInfoBlock(const MemProfSchema &Schema, const unsigned char *Ptr) { + deserialize(Schema, Ptr); + } + + // Read the contents of \p Ptr based on the \p Schema to populate the + // MemInfoBlock member. + void deserialize(const MemProfSchema &Schema, const unsigned char *Ptr) { + using namespace support; + + for (const Meta Id : Schema) { + switch (Id) { +#define MIBEntryDef(NameTag, Name, Type) \ + case Meta::Name: { \ + Name = endian::readNext<Type, little, unaligned>(Ptr); \ + } break; +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + default: + llvm_unreachable("Unknown meta type id, is the profile collected from " + "a newer version of the runtime?"); + } + } + } + + // Write the contents of the MemInfoBlock based on the \p Schema provided to + // the raw_ostream \p OS. + void serialize(const MemProfSchema &Schema, raw_ostream &OS) const { + using namespace support; + + endian::Writer LE(OS, little); + for (const Meta Id : Schema) { + switch (Id) { +#define MIBEntryDef(NameTag, Name, Type) \ + case Meta::Name: { \ + LE.write<Type>(Name); \ + } break; +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + default: + llvm_unreachable("Unknown meta type id, invalid input?"); + } + } + } + + // Print out the contents of the MemInfoBlock in YAML format. + void printYAML(raw_ostream &OS) const { + OS << " MemInfoBlock:\n"; +#define MIBEntryDef(NameTag, Name, Type) \ + OS << " " << #Name << ": " << Name << "\n"; +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + } + + // Define getters for each type which can be called by analyses. +#define MIBEntryDef(NameTag, Name, Type) \ + Type get##Name() const { return Name; } +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + + void clear() { *this = PortableMemInfoBlock(); } + + // Returns the full schema currently in use. + static MemProfSchema getSchema() { + MemProfSchema List; +#define MIBEntryDef(NameTag, Name, Type) List.push_back(Meta::Name); +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + return List; + } + + bool operator==(const PortableMemInfoBlock &Other) const { +#define MIBEntryDef(NameTag, Name, Type) \ + if (Other.get##Name() != get##Name()) \ + return false; +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + return true; + } + + bool operator!=(const PortableMemInfoBlock &Other) const { + return !operator==(Other); + } + + static constexpr size_t serializedSize() { + size_t Result = 0; +#define MIBEntryDef(NameTag, Name, Type) Result += sizeof(Type); +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef + return Result; + } + +private: +#define MIBEntryDef(NameTag, Name, Type) Type Name = Type(); +#include "llvm/ProfileData/MIBEntryDef.inc" +#undef MIBEntryDef +}; + +// A type representing the id generated by hashing the contents of the Frame. +using FrameId = uint64_t; +// Describes a call frame for a dynamic allocation context. The contents of +// the frame are populated by symbolizing the stack depot call frame from the +// compiler runtime. +struct Frame { + // A uuid (uint64_t) identifying the function. It is obtained by + // llvm::md5(FunctionName) which returns the lower 64 bits. + GlobalValue::GUID Function; + // The symbol name for the function. Only populated in the Frame by the reader + // if requested during initialization. This field should not be serialized. + llvm::Optional<std::string> SymbolName; + // The source line offset of the call from the beginning of parent function. + uint32_t LineOffset; + // The source column number of the call to help distinguish multiple calls + // on the same line. + uint32_t Column; + // Whether the current frame is inlined. + bool IsInlineFrame; + + Frame(const Frame &Other) { + Function = Other.Function; + SymbolName = Other.SymbolName; + LineOffset = Other.LineOffset; + Column = Other.Column; + IsInlineFrame = Other.IsInlineFrame; + } + + Frame(uint64_t Hash, uint32_t Off, uint32_t Col, bool Inline) + : Function(Hash), LineOffset(Off), Column(Col), IsInlineFrame(Inline) {} + + bool operator==(const Frame &Other) const { + // Ignore the SymbolName field to avoid a string compare. Comparing the + // function hash serves the same purpose. + return Other.Function == Function && Other.LineOffset == LineOffset && + Other.Column == Column && Other.IsInlineFrame == IsInlineFrame; + } + + Frame &operator=(const Frame &Other) { + Function = Other.Function; + SymbolName = Other.SymbolName; + LineOffset = Other.LineOffset; + Column = Other.Column; + IsInlineFrame = Other.IsInlineFrame; + return *this; + } + + bool operator!=(const Frame &Other) const { return !operator==(Other); } + + // Write the contents of the frame to the ostream \p OS. + void serialize(raw_ostream &OS) const { + using namespace support; + + endian::Writer LE(OS, little); + + // If the type of the GlobalValue::GUID changes, then we need to update + // the reader and the writer. + static_assert(std::is_same<GlobalValue::GUID, uint64_t>::value, + "Expect GUID to be uint64_t."); + LE.write<uint64_t>(Function); + + LE.write<uint32_t>(LineOffset); + LE.write<uint32_t>(Column); + LE.write<bool>(IsInlineFrame); + } + + // Read a frame from char data which has been serialized as little endian. + static Frame deserialize(const unsigned char *Ptr) { + using namespace support; + + const uint64_t F = endian::readNext<uint64_t, little, unaligned>(Ptr); + const uint32_t L = endian::readNext<uint32_t, little, unaligned>(Ptr); + const uint32_t C = endian::readNext<uint32_t, little, unaligned>(Ptr); + const bool I = endian::readNext<bool, little, unaligned>(Ptr); + return Frame(/*Function=*/F, /*LineOffset=*/L, /*Column=*/C, + /*IsInlineFrame=*/I); + } + + // Returns the size of the frame information. + static constexpr size_t serializedSize() { + return sizeof(Frame::Function) + sizeof(Frame::LineOffset) + + sizeof(Frame::Column) + sizeof(Frame::IsInlineFrame); + } + + // Print the frame information in YAML format. + void printYAML(raw_ostream &OS) const { + OS << " -\n" + << " Function: " << Function << "\n" + << " SymbolName: " << SymbolName.value_or("<None>") << "\n" + << " LineOffset: " << LineOffset << "\n" + << " Column: " << Column << "\n" + << " Inline: " << IsInlineFrame << "\n"; + } + + // Return a hash value based on the contents of the frame. Here we don't use + // hashing from llvm ADT since we are going to persist the hash id, the hash + // combine algorithm in ADT uses a new randomized seed each time. + inline FrameId hash() const { + auto HashCombine = [](auto Value, size_t Seed) { + std::hash<decltype(Value)> Hasher; + // The constant used below is the 64 bit representation of the fractional + // part of the golden ratio. Used here for the randomness in their bit + // pattern. + return Hasher(Value) + 0x9e3779b97f4a7c15 + (Seed << 6) + (Seed >> 2); + }; + + size_t Result = 0; + Result ^= HashCombine(Function, Result); + Result ^= HashCombine(LineOffset, Result); + Result ^= HashCombine(Column, Result); + Result ^= HashCombine(IsInlineFrame, Result); + return static_cast<FrameId>(Result); + } +}; + +// Holds allocation information in a space efficient format where frames are +// represented using unique identifiers. +struct IndexedAllocationInfo { + // The dynamic calling context for the allocation in bottom-up (leaf-to-root) + // order. Frame contents are stored out-of-line. + llvm::SmallVector<FrameId> CallStack; + // The statistics obtained from the runtime for the allocation. + PortableMemInfoBlock Info; + + IndexedAllocationInfo() = default; + IndexedAllocationInfo(ArrayRef<FrameId> CS, const MemInfoBlock &MB) + : CallStack(CS.begin(), CS.end()), Info(MB) {} + + // Returns the size in bytes when this allocation info struct is serialized. + size_t serializedSize() const { + return sizeof(uint64_t) + // The number of frames to serialize. + sizeof(FrameId) * CallStack.size() + // The callstack frame ids. + PortableMemInfoBlock::serializedSize(); // The size of the payload. + } + + bool operator==(const IndexedAllocationInfo &Other) const { + if (Other.Info != Info) + return false; + + if (Other.CallStack.size() != CallStack.size()) + return false; + + for (size_t J = 0; J < Other.CallStack.size(); J++) { + if (Other.CallStack[J] != CallStack[J]) + return false; + } + return true; + } + + bool operator!=(const IndexedAllocationInfo &Other) const { + return !operator==(Other); + } +}; + +// Holds allocation information with frame contents inline. The type should +// be used for temporary in-memory instances. +struct AllocationInfo { + // Same as IndexedAllocationInfo::CallStack with the frame contents inline. + llvm::SmallVector<Frame> CallStack; + // Same as IndexedAllocationInfo::Info; + PortableMemInfoBlock Info; + + AllocationInfo() = default; + AllocationInfo( + const IndexedAllocationInfo &IndexedAI, + llvm::function_ref<const Frame(const FrameId)> IdToFrameCallback) { + for (const FrameId &Id : IndexedAI.CallStack) { + CallStack.push_back(IdToFrameCallback(Id)); + } + Info = IndexedAI.Info; + } + + void printYAML(raw_ostream &OS) const { + OS << " -\n"; + OS << " Callstack:\n"; + // TODO: Print out the frame on one line with to make it easier for deep + // callstacks once we have a test to check valid YAML is generated. + for (const Frame &F : CallStack) { + F.printYAML(OS); + } + Info.printYAML(OS); + } +}; + +// Holds the memprof profile information for a function. The internal +// representation stores frame ids for efficiency. This representation should +// be used in the profile conversion and manipulation tools. +struct IndexedMemProfRecord { + // Memory allocation sites in this function for which we have memory + // profiling data. + llvm::SmallVector<IndexedAllocationInfo> AllocSites; + // Holds call sites in this function which are part of some memory + // allocation context. We store this as a list of locations, each with its + // list of inline locations in bottom-up order i.e. from leaf to root. The + // inline location list may include additional entries, users should pick + // the last entry in the list with the same function GUID. + llvm::SmallVector<llvm::SmallVector<FrameId>> CallSites; + + void clear() { + AllocSites.clear(); + CallSites.clear(); + } + + void merge(const IndexedMemProfRecord &Other) { + // TODO: Filter out duplicates which may occur if multiple memprof + // profiles are merged together using llvm-profdata. + AllocSites.append(Other.AllocSites); + CallSites.append(Other.CallSites); + } + + size_t serializedSize() const { + size_t Result = sizeof(GlobalValue::GUID); + for (const IndexedAllocationInfo &N : AllocSites) + Result += N.serializedSize(); + + // The number of callsites we have information for. + Result += sizeof(uint64_t); + for (const auto &Frames : CallSites) { + // The number of frame ids to serialize. + Result += sizeof(uint64_t); + Result += Frames.size() * sizeof(FrameId); + } + return Result; + } + + bool operator==(const IndexedMemProfRecord &Other) const { + if (Other.AllocSites.size() != AllocSites.size()) + return false; + + if (Other.CallSites.size() != CallSites.size()) + return false; + + for (size_t I = 0; I < AllocSites.size(); I++) { + if (AllocSites[I] != Other.AllocSites[I]) + return false; + } + + for (size_t I = 0; I < CallSites.size(); I++) { + if (CallSites[I] != Other.CallSites[I]) + return false; + } + return true; + } + + // Serializes the memprof records in \p Records to the ostream \p OS based + // on the schema provided in \p Schema. + void serialize(const MemProfSchema &Schema, raw_ostream &OS); + + // Deserializes memprof records from the Buffer. + static IndexedMemProfRecord deserialize(const MemProfSchema &Schema, + const unsigned char *Buffer); + + // Returns the GUID for the function name after canonicalization. For + // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are + // mapped to functions using this GUID. + static GlobalValue::GUID getGUID(const StringRef FunctionName); +}; + +// Holds the memprof profile information for a function. The internal +// representation stores frame contents inline. This representation should +// be used for small amount of temporary, in memory instances. +struct MemProfRecord { + // Same as IndexedMemProfRecord::AllocSites with frame contents inline. + llvm::SmallVector<AllocationInfo> AllocSites; + // Same as IndexedMemProfRecord::CallSites with frame contents inline. + llvm::SmallVector<llvm::SmallVector<Frame>> CallSites; + + MemProfRecord() = default; + MemProfRecord( + const IndexedMemProfRecord &Record, + llvm::function_ref<const Frame(const FrameId Id)> IdToFrameCallback) { + for (const IndexedAllocationInfo &IndexedAI : Record.AllocSites) { + AllocSites.emplace_back(IndexedAI, IdToFrameCallback); + } + for (const ArrayRef<FrameId> Site : Record.CallSites) { + llvm::SmallVector<Frame> Frames; + for (const FrameId Id : Site) { + Frames.push_back(IdToFrameCallback(Id)); + } + CallSites.push_back(Frames); + } + } + + // Prints out the contents of the memprof record in YAML. + void print(llvm::raw_ostream &OS) const { + if (!AllocSites.empty()) { + OS << " AllocSites:\n"; + for (const AllocationInfo &N : AllocSites) + N.printYAML(OS); + } + + if (!CallSites.empty()) { + OS << " CallSites:\n"; + for (const llvm::SmallVector<Frame> &Frames : CallSites) { + for (const Frame &F : Frames) { + OS << " -\n"; + F.printYAML(OS); + } + } + } + } +}; + +// Reads a memprof schema from a buffer. All entries in the buffer are +// interpreted as uint64_t. The first entry in the buffer denotes the number of +// ids in the schema. Subsequent entries are integers which map to memprof::Meta +// enum class entries. After successfully reading the schema, the pointer is one +// byte past the schema contents. +Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer); + +// Trait for reading IndexedMemProfRecord data from the on-disk hash table. +class RecordLookupTrait { +public: + using data_type = const IndexedMemProfRecord &; + using internal_key_type = uint64_t; + using external_key_type = uint64_t; + using hash_value_type = uint64_t; + using offset_type = uint64_t; + + RecordLookupTrait() = delete; + RecordLookupTrait(const MemProfSchema &S) : Schema(S) {} + + static bool EqualKey(uint64_t A, uint64_t B) { return A == B; } + static uint64_t GetInternalKey(uint64_t K) { return K; } + static uint64_t GetExternalKey(uint64_t K) { return K; } + + hash_value_type ComputeHash(uint64_t K) { return K; } + + static std::pair<offset_type, offset_type> + ReadKeyDataLength(const unsigned char *&D) { + using namespace support; + + offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D); + offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D); + return std::make_pair(KeyLen, DataLen); + } + + uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) { + using namespace support; + return endian::readNext<external_key_type, little, unaligned>(D); + } + + data_type ReadData(uint64_t K, const unsigned char *D, + offset_type /*Unused*/) { + Record = IndexedMemProfRecord::deserialize(Schema, D); + return Record; + } + +private: + // Holds the memprof schema used to deserialize records. + MemProfSchema Schema; + // Holds the records from one function deserialized from the indexed format. + IndexedMemProfRecord Record; +}; + +// Trait for writing IndexedMemProfRecord data to the on-disk hash table. +class RecordWriterTrait { +public: + using key_type = uint64_t; + using key_type_ref = uint64_t; + + using data_type = IndexedMemProfRecord; + using data_type_ref = IndexedMemProfRecord &; + + using hash_value_type = uint64_t; + using offset_type = uint64_t; + + // Pointer to the memprof schema to use for the generator. Unlike the reader + // we must use a default constructor with no params for the writer trait so we + // have a public member which must be initialized by the user. + MemProfSchema *Schema = nullptr; + + RecordWriterTrait() = default; + + static hash_value_type ComputeHash(key_type_ref K) { return K; } + + static std::pair<offset_type, offset_type> + EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { + using namespace support; + + endian::Writer LE(Out, little); + offset_type N = sizeof(K); + LE.write<offset_type>(N); + offset_type M = V.serializedSize(); + LE.write<offset_type>(M); + return std::make_pair(N, M); + } + + void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) { + using namespace support; + endian::Writer LE(Out, little); + LE.write<uint64_t>(K); + } + + void EmitData(raw_ostream &Out, key_type_ref /*Unused*/, data_type_ref V, + offset_type /*Unused*/) { + assert(Schema != nullptr && "MemProf schema is not initialized!"); + V.serialize(*Schema, Out); + } +}; + +// Trait for writing frame mappings to the on-disk hash table. +class FrameWriterTrait { +public: + using key_type = FrameId; + using key_type_ref = FrameId; + + using data_type = Frame; + using data_type_ref = Frame &; + + using hash_value_type = FrameId; + using offset_type = uint64_t; + + static hash_value_type ComputeHash(key_type_ref K) { return K; } + + static std::pair<offset_type, offset_type> + EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { + using namespace support; + endian::Writer LE(Out, little); + offset_type N = sizeof(K); + LE.write<offset_type>(N); + offset_type M = V.serializedSize(); + LE.write<offset_type>(M); + return std::make_pair(N, M); + } + + void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) { + using namespace support; + endian::Writer LE(Out, little); + LE.write<key_type>(K); + } + + void EmitData(raw_ostream &Out, key_type_ref /*Unused*/, data_type_ref V, + offset_type /*Unused*/) { + V.serialize(Out); + } +}; + +// Trait for reading frame mappings from the on-disk hash table. +class FrameLookupTrait { +public: + using data_type = const Frame; + using internal_key_type = FrameId; + using external_key_type = FrameId; + using hash_value_type = FrameId; + using offset_type = uint64_t; + + static bool EqualKey(internal_key_type A, internal_key_type B) { + return A == B; + } + static uint64_t GetInternalKey(internal_key_type K) { return K; } + static uint64_t GetExternalKey(external_key_type K) { return K; } + + hash_value_type ComputeHash(internal_key_type K) { return K; } + + static std::pair<offset_type, offset_type> + ReadKeyDataLength(const unsigned char *&D) { + using namespace support; + + offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D); + offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D); + return std::make_pair(KeyLen, DataLen); + } + + uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) { + using namespace support; + return endian::readNext<external_key_type, little, unaligned>(D); + } + + data_type ReadData(uint64_t K, const unsigned char *D, + offset_type /*Unused*/) { + return Frame::deserialize(D); + } +}; +} // namespace memprof +} // namespace llvm + +#endif // LLVM_PROFILEDATA_MEMPROF_H_ diff --git a/llvm/include/llvm/ProfileData/MemProfData.inc b/llvm/include/llvm/ProfileData/MemProfData.inc index ff22a697965c..6433cef84865 100644 --- a/llvm/include/llvm/ProfileData/MemProfData.inc +++ b/llvm/include/llvm/ProfileData/MemProfData.inc @@ -1,5 +1,5 @@ -#ifndef LLVM_PROFILEDATA_MEMPROFDATA_INC -#define LLVM_PROFILEDATA_MEMPROFDATA_INC +#ifndef MEMPROF_DATA_INC +#define MEMPROF_DATA_INC /*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\ |* |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. @@ -80,71 +80,90 @@ PACKED(struct SegmentEntry { } }); +// Packed struct definition for MSVC. We can't use the PACKED macro defined in +// MemProfData.inc since it would mean we are embedding a directive (the +// #include for MIBEntryDef) into the macros which is undefined behaviour. +#ifdef _MSC_VER +__pragma(pack(push,1)) +#endif + // A struct representing the heap allocation characteristics of a particular // runtime context. This struct is shared between the compiler-rt runtime and // the raw profile reader. The indexed format uses a separate, self-describing // backwards compatible format. -PACKED(struct MemInfoBlock { - uint32_t alloc_count; - uint64_t total_access_count, min_access_count, max_access_count; - uint64_t total_size; - uint32_t min_size, max_size; - uint32_t alloc_timestamp, dealloc_timestamp; - uint64_t total_lifetime; - uint32_t min_lifetime, max_lifetime; - uint32_t alloc_cpu_id, dealloc_cpu_id; - uint32_t num_migrated_cpu; - - // Only compared to prior deallocated object currently. - uint32_t num_lifetime_overlaps; - uint32_t num_same_alloc_cpu; - uint32_t num_same_dealloc_cpu; - - uint64_t data_type_id; // TODO: hash of type name - - MemInfoBlock() : alloc_count(0) {} - - MemInfoBlock(uint32_t size, uint64_t access_count, uint32_t alloc_timestamp, - uint32_t dealloc_timestamp, uint32_t alloc_cpu, uint32_t dealloc_cpu) - : alloc_count(1), total_access_count(access_count), - min_access_count(access_count), max_access_count(access_count), - total_size(size), min_size(size), max_size(size), - alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp), - total_lifetime(dealloc_timestamp - alloc_timestamp), - min_lifetime(total_lifetime), max_lifetime(total_lifetime), - alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu), - num_lifetime_overlaps(0), num_same_alloc_cpu(0), - num_same_dealloc_cpu(0) { - num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id; - } - - void Merge(const MemInfoBlock &newMIB) { - alloc_count += newMIB.alloc_count; - - total_access_count += newMIB.total_access_count; - min_access_count = newMIB.min_access_count < min_access_count ? newMIB.min_access_count : min_access_count; - max_access_count = newMIB.max_access_count < max_access_count ? newMIB.max_access_count : max_access_count; - - total_size += newMIB.total_size; - min_size = newMIB.min_size < min_size ? newMIB.min_size : min_size; - max_size = newMIB.max_size < max_size ? newMIB.max_size : max_size; +struct MemInfoBlock{ + +#define MIBEntryDef(NameTag, Name, Type) Type Name; +#include "MIBEntryDef.inc" +#undef MIBEntryDef + +bool operator==(const MemInfoBlock& Other) const { + bool IsEqual = true; +#define MIBEntryDef(NameTag, Name, Type) \ + IsEqual = (IsEqual && Name == Other.Name); +#include "MIBEntryDef.inc" +#undef MIBEntryDef + return IsEqual; +} + +MemInfoBlock() { +#define MIBEntryDef(NameTag, Name, Type) Name = Type(); +#include "MIBEntryDef.inc" +#undef MIBEntryDef +} + +MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs, + uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu) + : MemInfoBlock() { + AllocCount = 1U; + TotalAccessCount = AccessCount; + MinAccessCount = AccessCount; + MaxAccessCount = AccessCount; + TotalSize = Size; + MinSize = Size; + MaxSize = Size; + AllocTimestamp = AllocTs; + DeallocTimestamp = DeallocTs; + TotalLifetime = DeallocTimestamp - AllocTimestamp; + MinLifetime = TotalLifetime; + MaxLifetime = TotalLifetime; + AllocCpuId = AllocCpu; + DeallocCpuId = DeallocCpu; + NumMigratedCpu = AllocCpuId != DeallocCpuId; +} + +void Merge(const MemInfoBlock &newMIB) { + AllocCount += newMIB.AllocCount; + + TotalAccessCount += newMIB.TotalAccessCount; + MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount; + MaxAccessCount = newMIB.MaxAccessCount < MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount; + + TotalSize += newMIB.TotalSize; + MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize; + MaxSize = newMIB.MaxSize < MaxSize ? newMIB.MaxSize : MaxSize; + + TotalLifetime += newMIB.TotalLifetime; + MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime; + MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime; + + // We know newMIB was deallocated later, so just need to check if it was + // allocated before last one deallocated. + NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp; + AllocTimestamp = newMIB.AllocTimestamp; + DeallocTimestamp = newMIB.DeallocTimestamp; + + NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId; + NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId; + AllocCpuId = newMIB.AllocCpuId; + DeallocCpuId = newMIB.DeallocCpuId; +} - total_lifetime += newMIB.total_lifetime; - min_lifetime = newMIB.min_lifetime < min_lifetime ? newMIB.min_lifetime : min_lifetime; - max_lifetime = newMIB.max_lifetime > max_lifetime ? newMIB.max_lifetime : max_lifetime; - - // We know newMIB was deallocated later, so just need to check if it was - // allocated before last one deallocated. - num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp; - alloc_timestamp = newMIB.alloc_timestamp; - dealloc_timestamp = newMIB.dealloc_timestamp; - - num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id; - num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id; - alloc_cpu_id = newMIB.alloc_cpu_id; - dealloc_cpu_id = newMIB.dealloc_cpu_id; - } -}); +#ifdef _MSC_VER +} __pragma(pack(pop)); +#else +} __attribute__((__packed__)); +#endif } // namespace memprof } // namespace llvm diff --git a/llvm/include/llvm/ProfileData/RawMemProfReader.h b/llvm/include/llvm/ProfileData/RawMemProfReader.h index 45544927a86f..34f78063aa42 100644 --- a/llvm/include/llvm/ProfileData/RawMemProfReader.h +++ b/llvm/include/llvm/ProfileData/RawMemProfReader.h @@ -12,31 +12,142 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" +#include "llvm/DebugInfo/Symbolize/Symbolize.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/ProfileData/MemProf.h" +#include "llvm/ProfileData/MemProfData.inc" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" +#include <cstddef> + namespace llvm { namespace memprof { +// Map from id (recorded from sanitizer stack depot) to virtual addresses for +// each program counter address in the callstack. +using CallStackMap = llvm::DenseMap<uint64_t, llvm::SmallVector<uint64_t>>; + class RawMemProfReader { public: - RawMemProfReader(std::unique_ptr<MemoryBuffer> DataBuffer) - : DataBuffer(std::move(DataBuffer)) {} - // Prints aggregate counts for each raw profile parsed from the DataBuffer. - void printSummaries(raw_ostream &OS) const; + RawMemProfReader(const RawMemProfReader &) = delete; + RawMemProfReader &operator=(const RawMemProfReader &) = delete; + + // Prints the contents of the profile in YAML format. + void printYAML(raw_ostream &OS); // Return true if the \p DataBuffer starts with magic bytes indicating it is // a raw binary memprof profile. static bool hasFormat(const MemoryBuffer &DataBuffer); + // Return true if the file at \p Path starts with magic bytes indicating it is + // a raw binary memprof profile. + static bool hasFormat(const StringRef Path); // Create a RawMemProfReader after sanity checking the contents of the file at - // \p Path. - static Expected<std::unique_ptr<RawMemProfReader>> create(const Twine &Path); + // \p Path. The binary from which the profile has been collected is specified + // via a path in \p ProfiledBinary. + static Expected<std::unique_ptr<RawMemProfReader>> + create(const Twine &Path, const StringRef ProfiledBinary, + bool KeepName = false); + + using GuidMemProfRecordPair = std::pair<GlobalValue::GUID, MemProfRecord>; + using Iterator = InstrProfIterator<GuidMemProfRecordPair, RawMemProfReader>; + Iterator end() { return Iterator(); } + Iterator begin() { + Iter = FunctionProfileData.begin(); + return Iterator(this); + } + + Error readNextRecord(GuidMemProfRecordPair &GuidRecord); + + // The RawMemProfReader only holds memory profile information. + InstrProfKind getProfileKind() const { return InstrProfKind::MemProf; } + + // Constructor for unittests only. + RawMemProfReader(std::unique_ptr<llvm::symbolize::SymbolizableModule> Sym, + llvm::SmallVectorImpl<SegmentEntry> &Seg, + llvm::MapVector<uint64_t, MemInfoBlock> &Prof, + CallStackMap &SM, bool KeepName = false) + : Symbolizer(std::move(Sym)), SegmentInfo(Seg.begin(), Seg.end()), + CallstackProfileData(Prof), StackMap(SM), KeepSymbolName(KeepName) { + // We don't call initialize here since there is no raw profile to read. The + // test should pass in the raw profile as structured data. + + // If there is an error here then the mock symbolizer has not been + // initialized properly. + if (Error E = symbolizeAndFilterStackFrames()) + report_fatal_error(std::move(E)); + if (Error E = mapRawProfileToRecords()) + report_fatal_error(std::move(E)); + } + + // Return a const reference to the internal Id to Frame mappings. + const llvm::DenseMap<FrameId, Frame> &getFrameMapping() const { + return IdToFrame; + } + + // Return a const reference to the internal function profile data. + const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> & + getProfileData() const { + return FunctionProfileData; + } private: - std::unique_ptr<MemoryBuffer> DataBuffer; -}; + RawMemProfReader(object::OwningBinary<object::Binary> &&Bin, bool KeepName) + : Binary(std::move(Bin)), KeepSymbolName(KeepName) {} + // Initializes the RawMemProfReader with the contents in `DataBuffer`. + Error initialize(std::unique_ptr<MemoryBuffer> DataBuffer); + // Read and parse the contents of the `DataBuffer` as a binary format profile. + Error readRawProfile(std::unique_ptr<MemoryBuffer> DataBuffer); + // Symbolize and cache all the virtual addresses we encounter in the + // callstacks from the raw profile. Also prune callstack frames which we can't + // symbolize or those that belong to the runtime. For profile entries where + // the entire callstack is pruned, we drop the entry from the profile. + Error symbolizeAndFilterStackFrames(); + // Construct memprof records for each function and store it in the + // `FunctionProfileData` map. A function may have allocation profile data or + // callsite data or both. + Error mapRawProfileToRecords(); + + // A helper method to extract the frame from the IdToFrame map. + const Frame &idToFrame(const FrameId Id) const { + auto It = IdToFrame.find(Id); + assert(It != IdToFrame.end() && "Id not found in map."); + return It->getSecond(); + } + + object::SectionedAddress getModuleOffset(uint64_t VirtualAddress); + + object::OwningBinary<object::Binary> Binary; + std::unique_ptr<llvm::symbolize::SymbolizableModule> Symbolizer; + // The contents of the raw profile. + llvm::SmallVector<SegmentEntry, 16> SegmentInfo; + // A map from callstack id (same as key in CallStackMap below) to the heap + // information recorded for that allocation context. + llvm::MapVector<uint64_t, MemInfoBlock> CallstackProfileData; + CallStackMap StackMap; + + // Cached symbolization from PC to Frame. + llvm::DenseMap<uint64_t, llvm::SmallVector<FrameId>> SymbolizedFrame; + llvm::DenseMap<FrameId, Frame> IdToFrame; + + llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> FunctionProfileData; + llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord>::iterator Iter; + + // Whether to keep the symbol name for each frame after hashing. + bool KeepSymbolName = false; + // A mapping of the hash to symbol name, only used if KeepSymbolName is true. + llvm::DenseMap<uint64_t, std::string> GuidToSymbolName; +}; } // namespace memprof } // namespace llvm diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index bad2139fe8f0..f11392c05318 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -18,15 +18,12 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringSet.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/IR/Module.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cstdint> #include <list> @@ -40,6 +37,9 @@ namespace llvm { +class DILocation; +class raw_ostream; + const std::error_category &sampleprof_category(); enum class sampleprof_error { @@ -55,7 +55,6 @@ enum class sampleprof_error { not_implemented, counter_overflow, ostream_seek_unsupported, - compress_failed, uncompress_failed, zlib_unavailable, hash_mismatch @@ -201,9 +200,9 @@ enum class SecProfSummaryFlags : uint32_t { /// SecFlagFSDiscriminator means this profile uses flow-sensitive /// discriminators. SecFlagFSDiscriminator = (1 << 2), - /// SecFlagIsCSNested means this is context-sensitive nested profile for - /// CSSPGO - SecFlagIsCSNested = (1 << 4), + /// SecFlagIsPreInlined means this profile contains ShouldBeInlined + /// contexts thus this is CS preinliner computed. + SecFlagIsPreInlined = (1 << 4), }; enum class SecFuncMetadataFlags : uint32_t { @@ -343,6 +342,15 @@ public: : sampleprof_error::success; } + /// Decrease the number of samples for this record by \p S. Return the amout + /// of samples actually decreased. + uint64_t removeSamples(uint64_t S) { + if (S > NumSamples) + S = NumSamples; + NumSamples -= S; + return S; + } + /// Add called function \p F with samples \p S. /// Optionally scale sample count \p S by \p Weight. /// @@ -358,6 +366,18 @@ public: : sampleprof_error::success; } + /// Remove called function from the call target map. Return the target sample + /// count of the called function. + uint64_t removeCalledTarget(StringRef F) { + uint64_t Count = 0; + auto I = CallTargets.find(F); + if (I != CallTargets.end()) { + Count = I->second; + CallTargets.erase(I); + } + return Count; + } + /// Return true if this sample record contains function calls. bool hasCalls() const { return !CallTargets.empty(); } @@ -367,6 +387,13 @@ public: return SortCallTargets(CallTargets); } + uint64_t getCallTargetSum() const { + uint64_t Sum = 0; + for (const auto &I : CallTargets) + Sum += I.second; + return Sum; + } + /// Sort call targets in descending order of call frequency. static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) { SortedCallTargetSet SortedTargets; @@ -413,6 +440,8 @@ enum ContextAttributeMask { ContextNone = 0x0, ContextWasInlined = 0x1, // Leaf of context was inlined in previous build ContextShouldBeInlined = 0x2, // Leaf of context should be inlined + ContextDuplicatedIntoBase = + 0x4, // Leaf of context is duplicated into the base profile }; // Represents a context frame with function name and line location @@ -524,16 +553,6 @@ public: } } - // Promote context by removing top frames with the length of - // `ContextFramesToRemove`. Note that with array representation of context, - // the promotion is effectively a slice operation with first - // `ContextFramesToRemove` elements removed from left. - void promoteOnPath(uint32_t ContextFramesToRemove) { - assert(ContextFramesToRemove <= FullContext.size() && - "Cannot remove more than the whole context"); - FullContext = FullContext.drop_front(ContextFramesToRemove); - } - // Decode context string for a frame to get function name and location. // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. static void decodeContextString(StringRef ContextStr, StringRef &FName, @@ -703,6 +722,13 @@ public: : sampleprof_error::success; } + void removeTotalSamples(uint64_t Num) { + if (TotalSamples < Num) + TotalSamples = 0; + else + TotalSamples -= Num; + } + void setTotalSamples(uint64_t Num) { TotalSamples = Num; } sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) { @@ -727,6 +753,22 @@ public: FName, Num, Weight); } + // Remove a call target and decrease the body sample correspondingly. Return + // the number of body samples actually decreased. + uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset, + uint32_t Discriminator, + StringRef FName) { + uint64_t Count = 0; + auto I = BodySamples.find(LineLocation(LineOffset, Discriminator)); + if (I != BodySamples.end()) { + Count = I->second.removeCalledTarget(FName); + Count = I->second.removeSamples(Count); + if (!I->second.getSamples()) + BodySamples.erase(I); + } + return Count; + } + sampleprof_error addBodySamplesForProbe(uint32_t Index, uint64_t Num, uint64_t Weight = 1) { SampleRecord S; @@ -734,6 +776,19 @@ public: return BodySamples[LineLocation(Index, 0)].merge(S, Weight); } + // Accumulate all call target samples to update the body samples. + void updateCallsiteSamples() { + for (auto &I : BodySamples) { + uint64_t TargetSamples = I.second.getCallTargetSum(); + // It's possible that the body sample count can be greater than the call + // target sum. E.g, if some call targets are external targets, they won't + // be considered valid call targets, but the body sample count which is + // from lbr ranges can actually include them. + if (TargetSamples > I.second.getSamples()) + I.second.addSamples(TargetSamples - I.second.getSamples()); + } + } + // Accumulate all body samples to set total samples. void updateTotalSamples() { setTotalSamples(0); @@ -829,7 +884,7 @@ public: /// Return the sample count of the first instruction of the function. /// The function can be either a standalone symbol or an inlined function. uint64_t getEntrySamples() const { - if (FunctionSamples::ProfileIsCSFlat && getHeadSamples()) { + if (FunctionSamples::ProfileIsCS && getHeadSamples()) { // For CS profile, if we already have more accurate head samples // counted by branch sample from caller, use them as entry samples. return getHeadSamples(); @@ -1046,16 +1101,14 @@ public: static bool ProfileIsProbeBased; - static bool ProfileIsCSFlat; + static bool ProfileIsCS; - static bool ProfileIsCSNested; + static bool ProfileIsPreInlined; SampleContext &getContext() const { return Context; } void setContext(const SampleContext &FContext) { Context = FContext; } - static SampleProfileFormat Format; - /// Whether the profile uses MD5 to represent string. static bool UseMD5; diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index a2caca246d93..7da336b9f61b 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -227,10 +227,8 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/DiagnosticInfo.h" -#include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/GCOV.h" @@ -240,7 +238,6 @@ #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SymbolRemappingReader.h" -#include <algorithm> #include <cstdint> #include <list> #include <memory> @@ -473,11 +470,11 @@ public: /// Whether input profile is based on pseudo probes. bool profileIsProbeBased() const { return ProfileIsProbeBased; } - /// Whether input profile is fully context-sensitive and flat. - bool profileIsCSFlat() const { return ProfileIsCSFlat; } + /// Whether input profile is fully context-sensitive. + bool profileIsCS() const { return ProfileIsCS; } - /// Whether input profile is fully context-sensitive and nested. - bool profileIsCSNested() const { return ProfileIsCSNested; } + /// Whether input profile contains ShouldBeInlined contexts. + bool profileIsPreInlined() const { return ProfileIsPreInlined; } virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() { return nullptr; @@ -537,10 +534,10 @@ protected: bool ProfileIsProbeBased = false; /// Whether function profiles are context-sensitive flat profiles. - bool ProfileIsCSFlat = false; + bool ProfileIsCS = false; - /// Whether function profiles are context-sensitive nested profiles. - bool ProfileIsCSNested = false; + /// Whether function profile contains ShouldBeInlined contexts. + bool ProfileIsPreInlined = false; /// Number of context-sensitive profiles. uint32_t CSProfileCount = 0; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index 42decd255203..aa7f1cbdd7e8 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -13,19 +13,15 @@ #define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H #include "llvm/ADT/MapVector.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringSet.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cstdint> #include <memory> #include <set> #include <system_error> -#include <unordered_set> namespace llvm { namespace sampleprof { diff --git a/llvm/include/llvm/Remarks/RemarkSerializer.h b/llvm/include/llvm/Remarks/RemarkSerializer.h index 6217bd98d1a5..b971173ad2c6 100644 --- a/llvm/include/llvm/Remarks/RemarkSerializer.h +++ b/llvm/include/llvm/Remarks/RemarkSerializer.h @@ -13,7 +13,6 @@ #ifndef LLVM_REMARKS_REMARKSERIALIZER_H #define LLVM_REMARKS_REMARKSERIALIZER_H -#include "llvm/Remarks/Remark.h" #include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkStringTable.h" diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def index a953e9439db4..e2f949856d9f 100644 --- a/llvm/include/llvm/Support/AArch64TargetParser.def +++ b/llvm/include/llvm/Support/AArch64TargetParser.def @@ -168,10 +168,10 @@ AARCH64_CPU_NAME("cortex-a510", ARMV9A, FK_NEON_FP_ARMV8, false, AARCH64_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a65", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RCPC | AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cortex-a65ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RCPC | AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) @@ -190,10 +190,11 @@ AARCH64_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | - AArch64::AEK_SSBS)) + AArch64::AEK_SSBS | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("cortex-a78c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | - AArch64::AEK_SSBS)) + AArch64::AEK_SSBS | AArch64::AEK_PROFILE | AArch64::AEK_FLAGM | + AArch64::AEK_PAUTH | AArch64::AEK_FP16FML)) AARCH64_CPU_NAME("cortex-a710", ARMV9A, FK_NEON_FP_ARMV8, false, (AArch64::AEK_MTE | AArch64::AEK_PAUTH | AArch64::AEK_FLAGM | AArch64::AEK_SB | AArch64::AEK_I8MM | AArch64::AEK_FP16FML | @@ -203,35 +204,37 @@ AARCH64_CPU_NAME("cortex-r82", ARMV8R, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_LSE)) AARCH64_CPU_NAME("cortex-x1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | - AArch64::AEK_SSBS)) + AArch64::AEK_SSBS | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("cortex-x1c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | - AArch64::AEK_SSBS | AArch64::AEK_PAUTH)) + AArch64::AEK_SSBS | AArch64::AEK_PAUTH | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("cortex-x2", ARMV9A, FK_NEON_FP_ARMV8, false, (AArch64::AEK_MTE | AArch64::AEK_BF16 | AArch64::AEK_I8MM | AArch64::AEK_PAUTH | AArch64::AEK_SSBS | AArch64::AEK_SB | AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM | AArch64::AEK_FP16FML)) AARCH64_CPU_NAME("neoverse-e1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS | + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RCPC | AArch64::AEK_SSBS)) AARCH64_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | - AArch64::AEK_PROFILE | AArch64::AEK_RAS | AArch64::AEK_RCPC | + AArch64::AEK_PROFILE | AArch64::AEK_RCPC | AArch64::AEK_SSBS)) AARCH64_CPU_NAME("neoverse-n2", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_BF16 | AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | - AArch64::AEK_I8MM | AArch64::AEK_MTE | AArch64::AEK_RAS | - AArch64::AEK_RCPC | AArch64::AEK_SB | AArch64::AEK_SSBS | + AArch64::AEK_I8MM | AArch64::AEK_MTE | + AArch64::AEK_SB | AArch64::AEK_SSBS | AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM)) AARCH64_CPU_NAME("neoverse-512tvb", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_RAS | AArch64::AEK_SVE | AArch64::AEK_SSBS | - AArch64::AEK_RCPC | AArch64::AEK_FP16 | AArch64::AEK_BF16 | - AArch64::AEK_DOTPROD )) + (AArch64::AEK_SVE | AArch64::AEK_SSBS | + AArch64::AEK_FP16 | AArch64::AEK_BF16 | + AArch64::AEK_DOTPROD | AArch64::AEK_PROFILE | + AArch64::AEK_RAND | AArch64::AEK_FP16FML | AArch64::AEK_I8MM)) AARCH64_CPU_NAME("neoverse-v1", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_RAS | AArch64::AEK_SVE | AArch64::AEK_SSBS | - AArch64::AEK_RCPC | AArch64::AEK_FP16 | AArch64::AEK_BF16 | - AArch64::AEK_DOTPROD )) + (AArch64::AEK_SVE | AArch64::AEK_SSBS | + AArch64::AEK_FP16 | AArch64::AEK_BF16 | + AArch64::AEK_DOTPROD | AArch64::AEK_PROFILE | + AArch64::AEK_RAND | AArch64::AEK_FP16FML | AArch64::AEK_I8MM)) AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) AARCH64_CPU_NAME("apple-a7", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, @@ -247,11 +250,11 @@ AARCH64_CPU_NAME("apple-a11", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, AARCH64_CPU_NAME("apple-a12", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16)) AARCH64_CPU_NAME("apple-a13", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_FP16 | AArch64::AEK_FP16FML)) + (AArch64::AEK_FP16 | AArch64::AEK_FP16FML | AArch64::AEK_SHA3)) AARCH64_CPU_NAME("apple-a14", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_FP16 | AArch64::AEK_FP16FML)) + (AArch64::AEK_FP16 | AArch64::AEK_FP16FML | AArch64::AEK_SHA3)) AARCH64_CPU_NAME("apple-m1", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_FP16 | AArch64::AEK_FP16FML)) + (AArch64::AEK_FP16 | AArch64::AEK_FP16FML | AArch64::AEK_SHA3)) AARCH64_CPU_NAME("apple-s4", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16)) AARCH64_CPU_NAME("apple-s5", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, @@ -271,17 +274,15 @@ AARCH64_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, AARCH64_CPU_NAME("thunderx2t99", ARMV8_1A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) AARCH64_CPU_NAME("thunderx3t110", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AEK_CRYPTO | AEK_FP | AEK_SIMD | - AEK_LSE | AEK_RAND | AArch64::AEK_PROFILE | - AArch64::AEK_RAS)) + (AArch64::AEK_NONE)) AARCH64_CPU_NAME("thunderx", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("thunderxt88", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("thunderxt81", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("thunderxt83", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("tsv110", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_FP16FML | @@ -290,6 +291,8 @@ AARCH64_CPU_NAME("a64fx", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_SVE)) AARCH64_CPU_NAME("carmel", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, AArch64::AEK_FP16) +AARCH64_CPU_NAME("ampere1", ARMV8_6A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_SB | AArch64::AEK_SSBS)) // Invalid CPU AARCH64_CPU_NAME("invalid", INVALID, FK_INVALID, true, AArch64::AEK_INVALID) #undef AARCH64_CPU_NAME diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h index aec80291f01f..41d144cfd5c4 100644 --- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h +++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h @@ -136,13 +136,17 @@ enum : int32_t { // Compute program resource register 3 for GFX10+. Must match hardware // definition. -#define COMPUTE_PGM_RSRC3_GFX10(NAME, SHIFT, WIDTH) \ - AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_ ## NAME, SHIFT, WIDTH) +#define COMPUTE_PGM_RSRC3_GFX10_PLUS(NAME, SHIFT, WIDTH) \ + AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_PLUS_ ## NAME, SHIFT, WIDTH) enum : int32_t { - COMPUTE_PGM_RSRC3_GFX10(SHARED_VGPR_COUNT, 0, 4), // GFX10+ - COMPUTE_PGM_RSRC3_GFX10(RESERVED0, 4, 28), + COMPUTE_PGM_RSRC3_GFX10_PLUS(SHARED_VGPR_COUNT, 0, 4), // GFX10+ + COMPUTE_PGM_RSRC3_GFX10_PLUS(INST_PREF_SIZE, 4, 6), // GFX11+ + COMPUTE_PGM_RSRC3_GFX10_PLUS(TRAP_ON_START, 10, 1), // GFX11+ + COMPUTE_PGM_RSRC3_GFX10_PLUS(TRAP_ON_END, 11, 1), // GFX11+ + COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED0, 12, 19), + COMPUTE_PGM_RSRC3_GFX10_PLUS(IMAGE_OP, 31, 1), // GFX11+ }; -#undef COMPUTE_PGM_RSRC3_GFX10 +#undef COMPUTE_PGM_RSRC3_GFX10_PLUS // Kernel code properties. Must be kept backwards compatible. #define KERNEL_CODE_PROPERTY(NAME, SHIFT, WIDTH) \ diff --git a/llvm/include/llvm/Support/ARMBuildAttributes.h b/llvm/include/llvm/Support/ARMBuildAttributes.h index b4405e7d4908..35f8992ca932 100644 --- a/llvm/include/llvm/Support/ARMBuildAttributes.h +++ b/llvm/include/llvm/Support/ARMBuildAttributes.h @@ -90,25 +90,26 @@ enum AttrType : unsigned { // Legal Values for CPU_arch, (=6), uleb128 enum CPUArch { - Pre_v4 = 0, - v4 = 1, // e.g. SA110 - v4T = 2, // e.g. ARM7TDMI - v5T = 3, // e.g. ARM9TDMI - v5TE = 4, // e.g. ARM946E_S - v5TEJ = 5, // e.g. ARM926EJ_S - v6 = 6, // e.g. ARM1136J_S - v6KZ = 7, // e.g. ARM1176JZ_S - v6T2 = 8, // e.g. ARM1156T2_S - v6K = 9, // e.g. ARM1176JZ_S - v7 = 10, // e.g. Cortex A8, Cortex M3 - v6_M = 11, // e.g. Cortex M1 - v6S_M = 12, // v6_M with the System extensions - v7E_M = 13, // v7_M with DSP extensions - v8_A = 14, // v8_A AArch32 - v8_R = 15, // e.g. Cortex R52 - v8_M_Base= 16, // v8_M_Base AArch32 - v8_M_Main= 17, // v8_M_Main AArch32 - v8_1_M_Main=21, // v8_1_M_Main AArch32 + Pre_v4 = 0, + v4 = 1, // e.g. SA110 + v4T = 2, // e.g. ARM7TDMI + v5T = 3, // e.g. ARM9TDMI + v5TE = 4, // e.g. ARM946E_S + v5TEJ = 5, // e.g. ARM926EJ_S + v6 = 6, // e.g. ARM1136J_S + v6KZ = 7, // e.g. ARM1176JZ_S + v6T2 = 8, // e.g. ARM1156T2_S + v6K = 9, // e.g. ARM1176JZ_S + v7 = 10, // e.g. Cortex A8, Cortex M3 + v6_M = 11, // e.g. Cortex M1 + v6S_M = 12, // v6_M with the System extensions + v7E_M = 13, // v7_M with DSP extensions + v8_A = 14, // v8_A AArch32 + v8_R = 15, // e.g. Cortex R52 + v8_M_Base = 16, // v8_M_Base AArch32 + v8_M_Main = 17, // v8_M_Main AArch32 + v8_1_M_Main = 21, // v8_1_M_Main AArch32 + v9_A = 22, // v9_A AArch32 }; enum CPUArchProfile { // (=7), uleb128 diff --git a/llvm/include/llvm/Support/ARMTargetParser.def b/llvm/include/llvm/Support/ARMTargetParser.def index 80deeb2a6e9d..6a1ac7213dad 100644 --- a/llvm/include/llvm/Support/ARMTargetParser.def +++ b/llvm/include/llvm/Support/ARMTargetParser.def @@ -129,22 +129,22 @@ ARM_ARCH("armv8.8-a", ARMV8_8A, "8.8-A", "v8.8a", ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_SHA2 | ARM::AEK_AES | ARM::AEK_I8MM)) ARM_ARCH("armv9-a", ARMV9A, "9-A", "v9a", - ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8, + ARMBuildAttrs::CPUArch::v9_A, FK_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD)) ARM_ARCH("armv9.1-a", ARMV9_1A, "9.1-A", "v9.1a", - ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8, + ARMBuildAttrs::CPUArch::v9_A, FK_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) ARM_ARCH("armv9.2-a", ARMV9_2A, "9.2-A", "v9.2a", - ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8, + ARMBuildAttrs::CPUArch::v9_A, FK_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) ARM_ARCH("armv9.3-a", ARMV9_3A, "9.3-A", "v9.3a", - ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + ARMBuildAttrs::CPUArch::v9_A, FK_CRYPTO_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) diff --git a/llvm/include/llvm/Support/ARMWinEH.h b/llvm/include/llvm/Support/ARMWinEH.h index 327aa9804849..dee2f31fb127 100644 --- a/llvm/include/llvm/Support/ARMWinEH.h +++ b/llvm/include/llvm/Support/ARMWinEH.h @@ -199,13 +199,14 @@ inline bool EpilogueFolding(const RuntimeFunction &RF) { inline uint16_t StackAdjustment(const RuntimeFunction &RF) { uint16_t Adjustment = RF.StackAdjust(); if (Adjustment >= 0x3f4) - return (Adjustment & 0x3) ? ((Adjustment & 0x3) << 2) - 1 : 0; + return (Adjustment & 0x3) + 1; return Adjustment; } /// SavedRegisterMask - Utility function to calculate the set of saved general /// purpose (r0-r15) and VFP (d0-d31) registers. -std::pair<uint16_t, uint32_t> SavedRegisterMask(const RuntimeFunction &RF); +std::pair<uint16_t, uint32_t> SavedRegisterMask(const RuntimeFunction &RF, + bool Prologue = true); /// RuntimeFunctionARM64 - An entry in the table of procedure data (.pdata) /// diff --git a/llvm/include/llvm/Support/Alignment.h b/llvm/include/llvm/Support/Alignment.h index 1176c026ba99..1543a5713d73 100644 --- a/llvm/include/llvm/Support/Alignment.h +++ b/llvm/include/llvm/Support/Alignment.h @@ -84,6 +84,14 @@ public: /// Needed to interact with C for instance. uint64_t value() const { return uint64_t(1) << ShiftValue; } + // Returns the previous alignment. + Align previous() const { + assert(ShiftValue != 0 && "Undefined operation"); + Align Out; + Out.ShiftValue = ShiftValue - 1; + return Out; + } + /// Allow constructions of constexpr Align. template <size_t kValue> constexpr static LogValue Constant() { return LogValue{static_cast<uint8_t>(CTLog2<kValue>())}; @@ -131,7 +139,7 @@ public: } /// For convenience, returns a valid alignment or 1 if undefined. - Align valueOrOne() const { return hasValue() ? getValue() : Align(); } + Align valueOrOne() const { return value_or(Align()); } }; /// Checks that SizeInBytes is a multiple of the alignment. @@ -173,13 +181,7 @@ inline uint64_t alignTo(uint64_t Size, Align A) { inline uint64_t alignTo(uint64_t Size, Align A, uint64_t Skew) { const uint64_t Value = A.value(); Skew %= Value; - return ((Size + Value - 1 - Skew) & ~(Value - 1U)) + Skew; -} - -/// Returns a multiple of A needed to store `Size` bytes. -/// Returns `Size` if current alignment is undefined. -inline uint64_t alignTo(uint64_t Size, MaybeAlign A) { - return A ? alignTo(Size, A.getValue()) : Size; + return alignTo(Size - Skew, A) + Skew; } /// Aligns `Addr` to `Alignment` bytes, rounding up. @@ -208,26 +210,10 @@ inline unsigned Log2(Align A) { return A.ShiftValue; } /// Returns the alignment that satisfies both alignments. /// Same semantic as MinAlign. -inline Align commonAlignment(Align A, Align B) { return std::min(A, B); } - -/// Returns the alignment that satisfies both alignments. -/// Same semantic as MinAlign. inline Align commonAlignment(Align A, uint64_t Offset) { return Align(MinAlign(A.value(), Offset)); } -/// Returns the alignment that satisfies both alignments. -/// Same semantic as MinAlign. -inline MaybeAlign commonAlignment(MaybeAlign A, MaybeAlign B) { - return A && B ? commonAlignment(*A, *B) : A ? A : B; -} - -/// Returns the alignment that satisfies both alignments. -/// Same semantic as MinAlign. -inline MaybeAlign commonAlignment(MaybeAlign A, uint64_t Offset) { - return MaybeAlign(MinAlign((*A).value(), Offset)); -} - /// Returns a representation of the alignment that encodes undefined as 0. inline unsigned encode(MaybeAlign A) { return A ? A->ShiftValue + 1 : 0; } @@ -270,14 +256,6 @@ inline bool operator>(Align Lhs, uint64_t Rhs) { return Lhs.value() > Rhs; } -/// Comparisons between MaybeAlign and scalars. -inline bool operator==(MaybeAlign Lhs, uint64_t Rhs) { - return Lhs ? (*Lhs).value() == Rhs : Rhs == 0; -} -inline bool operator!=(MaybeAlign Lhs, uint64_t Rhs) { - return Lhs ? (*Lhs).value() != Rhs : Rhs != 0; -} - /// Comparisons operators between Align. inline bool operator==(Align Lhs, Align Rhs) { return Lhs.ShiftValue == Rhs.ShiftValue; @@ -314,37 +292,6 @@ bool operator>=(MaybeAlign Lhs, MaybeAlign Rhs) = delete; bool operator<(MaybeAlign Lhs, MaybeAlign Rhs) = delete; bool operator>(MaybeAlign Lhs, MaybeAlign Rhs) = delete; -inline Align operator*(Align Lhs, uint64_t Rhs) { - assert(Rhs > 0 && "Rhs must be positive"); - return Align(Lhs.value() * Rhs); -} - -inline MaybeAlign operator*(MaybeAlign Lhs, uint64_t Rhs) { - assert(Rhs > 0 && "Rhs must be positive"); - return Lhs ? Lhs.getValue() * Rhs : MaybeAlign(); -} - -inline Align operator/(Align Lhs, uint64_t Divisor) { - assert(llvm::isPowerOf2_64(Divisor) && - "Divisor must be positive and a power of 2"); - assert(Lhs != 1 && "Can't halve byte alignment"); - return Align(Lhs.value() / Divisor); -} - -inline MaybeAlign operator/(MaybeAlign Lhs, uint64_t Divisor) { - assert(llvm::isPowerOf2_64(Divisor) && - "Divisor must be positive and a power of 2"); - return Lhs ? Lhs.getValue() / Divisor : MaybeAlign(); -} - -inline Align max(MaybeAlign Lhs, Align Rhs) { - return Lhs && *Lhs > Rhs ? *Lhs : Rhs; -} - -inline Align max(Align Lhs, MaybeAlign Rhs) { - return Rhs && *Rhs > Lhs ? *Rhs : Lhs; -} - #ifndef NDEBUG // For usage in LLVM_DEBUG macros. inline std::string DebugStr(const Align &A) { diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h index ec5ed06b7fa4..5ca0c9decac3 100644 --- a/llvm/include/llvm/Support/Allocator.h +++ b/llvm/include/llvm/Support/Allocator.h @@ -140,6 +140,9 @@ public: // This method is *not* marked noalias, because // SpecificBumpPtrAllocator::DestroyAll() loops over all allocations, and // that loop is not based on the Allocate() return value. + // + // Allocate(0, N) is valid, it returns a non-null pointer (which should not + // be dereferenced). LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, Align Alignment) { // Keep track of how many bytes we've allocated. BytesAllocated += Size; @@ -154,7 +157,9 @@ public: #endif // Check if we have enough space. - if (Adjustment + SizeToAllocate <= size_t(End - CurPtr)) { + if (Adjustment + SizeToAllocate <= size_t(End - CurPtr) + // We can't return nullptr even for a zero-sized allocation! + && CurPtr != nullptr) { char *AlignedPtr = CurPtr + Adjustment; CurPtr = AlignedPtr + SizeToAllocate; // Update the allocation point of this memory block in MemorySanitizer. diff --git a/llvm/include/llvm/Support/BLAKE3.h b/llvm/include/llvm/Support/BLAKE3.h new file mode 100644 index 000000000000..7b30dbccd173 --- /dev/null +++ b/llvm/include/llvm/Support/BLAKE3.h @@ -0,0 +1,124 @@ +//==- BLAKE3.h - BLAKE3 C++ wrapper for LLVM ---------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This is a C++ wrapper of the BLAKE3 C interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_BLAKE3_H +#define LLVM_SUPPORT_BLAKE3_H + +#include "llvm-c/blake3.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +namespace llvm { + +/// The constant \p LLVM_BLAKE3_OUT_LEN provides the default output length, +/// 32 bytes, which is recommended for most callers. +/// +/// Outputs shorter than the default length of 32 bytes (256 bits) provide +/// less security. An N-bit BLAKE3 output is intended to provide N bits of +/// first and second preimage resistance and N/2 bits of collision +/// resistance, for any N up to 256. Longer outputs don't provide any +/// additional security. +/// +/// Shorter BLAKE3 outputs are prefixes of longer ones. Explicitly +/// requesting a short output is equivalent to truncating the default-length +/// output. +template <size_t NumBytes = LLVM_BLAKE3_OUT_LEN> +using BLAKE3Result = std::array<uint8_t, NumBytes>; + +/// A class that wraps the BLAKE3 algorithm. +class BLAKE3 { +public: + BLAKE3() { init(); } + + /// Reinitialize the internal state + void init() { llvm_blake3_hasher_init(&Hasher); } + + /// Digest more data. + void update(ArrayRef<uint8_t> Data) { + llvm_blake3_hasher_update(&Hasher, Data.data(), Data.size()); + } + + /// Digest more data. + void update(StringRef Str) { + llvm_blake3_hasher_update(&Hasher, Str.data(), Str.size()); + } + + /// Finalize the hasher and put the result in \p Result. + /// This doesn't modify the hasher itself, and it's possible to finalize again + /// after adding more input. + template <size_t NumBytes = LLVM_BLAKE3_OUT_LEN> + void final(BLAKE3Result<NumBytes> &Result) { + llvm_blake3_hasher_finalize(&Hasher, Result.data(), Result.size()); + } + + /// Finalize the hasher and return an output of any length, given in bytes. + /// This doesn't modify the hasher itself, and it's possible to finalize again + /// after adding more input. + template <size_t NumBytes = LLVM_BLAKE3_OUT_LEN> + BLAKE3Result<NumBytes> final() { + BLAKE3Result<NumBytes> Result; + llvm_blake3_hasher_finalize(&Hasher, Result.data(), Result.size()); + return Result; + } + + /// Return the current output for the digested data since the last call to + /// init(). + /// + /// Other hash functions distinguish between \p result() and \p final(), with + /// \p result() allowing more calls into \p update(), but there's no + // difference for the BLAKE3 hash function. + template <size_t NumBytes = LLVM_BLAKE3_OUT_LEN> + BLAKE3Result<NumBytes> result() { + return final<NumBytes>(); + } + + /// Returns a BLAKE3 hash for the given data. + template <size_t NumBytes = LLVM_BLAKE3_OUT_LEN> + static BLAKE3Result<NumBytes> hash(ArrayRef<uint8_t> Data) { + BLAKE3 Hasher; + Hasher.update(Data); + return Hasher.final<NumBytes>(); + } + +private: + llvm_blake3_hasher Hasher; +}; + +/// Like \p BLAKE3 but using a class-level template parameter for specifying the +/// hash size of the \p final() and \p result() functions. +/// +/// This is useful for using BLAKE3 as the hasher type for \p HashBuilder with +/// non-default hash sizes. +template <size_t NumBytes> class TruncatedBLAKE3 : public BLAKE3 { +public: + /// Finalize the hasher and put the result in \p Result. + /// This doesn't modify the hasher itself, and it's possible to finalize again + /// after adding more input. + void final(BLAKE3Result<NumBytes> &Result) { return BLAKE3::final(Result); } + + /// Finalize the hasher and return an output of any length, given in bytes. + /// This doesn't modify the hasher itself, and it's possible to finalize again + /// after adding more input. + BLAKE3Result<NumBytes> final() { return BLAKE3::final<NumBytes>(); } + + /// Return the current output for the digested data since the last call to + /// init(). + /// + /// Other hash functions distinguish between \p result() and \p final(), with + /// \p result() allowing more calls into \p update(), but there's no + // difference for the BLAKE3 hash function. + BLAKE3Result<NumBytes> result() { return BLAKE3::result<NumBytes>(); } +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/Base64.h b/llvm/include/llvm/Support/Base64.h index 62064a35aa34..da4ae1688574 100644 --- a/llvm/include/llvm/Support/Base64.h +++ b/llvm/include/llvm/Support/Base64.h @@ -13,6 +13,7 @@ #ifndef LLVM_SUPPORT_BASE64_H #define LLVM_SUPPORT_BASE64_H +#include <cstdint> #include <string> namespace llvm { diff --git a/llvm/include/llvm/Support/BinaryStreamArray.h b/llvm/include/llvm/Support/BinaryStreamArray.h index c3e0db4dcff0..ef2233c53ec2 100644 --- a/llvm/include/llvm/Support/BinaryStreamArray.h +++ b/llvm/include/llvm/Support/BinaryStreamArray.h @@ -111,6 +111,8 @@ public: bool valid() const { return Stream.valid(); } + bool isOffsetValid(uint32_t Offset) const { return at(Offset) != end(); } + uint32_t skew() const { return Skew; } Iterator end() const { return Iterator(E); } diff --git a/llvm/include/llvm/Support/BinaryStreamRef.h b/llvm/include/llvm/Support/BinaryStreamRef.h index bc8c6a496ecf..46fc9fb293df 100644 --- a/llvm/include/llvm/Support/BinaryStreamRef.h +++ b/llvm/include/llvm/Support/BinaryStreamRef.h @@ -48,7 +48,7 @@ public: } uint64_t getLength() const { - if (Length.hasValue()) + if (Length) return *Length; return BorrowedImpl ? (BorrowedImpl->getLength() - ViewOffset) : 0; @@ -67,7 +67,7 @@ public: return Result; Result.ViewOffset += N; - if (Result.Length.hasValue()) + if (Result.Length) *Result.Length -= N; return Result; } @@ -87,7 +87,7 @@ public: // Since we're dropping non-zero bytes from the end, stop length-tracking // by setting the length of the resulting StreamRef to an explicit value. - if (!Result.Length.hasValue()) + if (!Result.Length) Result.Length = getLength(); *Result.Length -= N; diff --git a/llvm/include/llvm/Support/BranchProbability.h b/llvm/include/llvm/Support/BranchProbability.h index 6f071c15421f..79d70cf611d4 100644 --- a/llvm/include/llvm/Support/BranchProbability.h +++ b/llvm/include/llvm/Support/BranchProbability.h @@ -16,6 +16,7 @@ #include "llvm/Support/DataTypes.h" #include <algorithm> #include <cassert> +#include <iterator> #include <numeric> namespace llvm { diff --git a/llvm/include/llvm/Support/CSKYAttributeParser.h b/llvm/include/llvm/Support/CSKYAttributeParser.h new file mode 100644 index 000000000000..e926ebe5e306 --- /dev/null +++ b/llvm/include/llvm/Support/CSKYAttributeParser.h @@ -0,0 +1,43 @@ +//===---- CSKYAttributeParser.h - CSKY Attribute Parser ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CSKYATTRIBUTEPARSER_H +#define LLVM_SUPPORT_CSKYATTRIBUTEPARSER_H + +#include "llvm/Support/CSKYAttributes.h" +#include "llvm/Support/ELFAttributeParser.h" + +namespace llvm { +class CSKYAttributeParser : public ELFAttributeParser { + struct DisplayHandler { + CSKYAttrs::AttrType attribute; + Error (CSKYAttributeParser::*routine)(unsigned); + }; + static const DisplayHandler displayRoutines[]; + + Error dspVersion(unsigned tag); + Error vdspVersion(unsigned tag); + Error fpuVersion(unsigned tag); + Error fpuABI(unsigned tag); + Error fpuRounding(unsigned tag); + Error fpuDenormal(unsigned tag); + Error fpuException(unsigned tag); + Error fpuHardFP(unsigned tag); + + Error handler(uint64_t tag, bool &handled) override; + +public: + CSKYAttributeParser(ScopedPrinter *sw) + : ELFAttributeParser(sw, CSKYAttrs::getCSKYAttributeTags(), "csky") {} + CSKYAttributeParser() + : ELFAttributeParser(CSKYAttrs::getCSKYAttributeTags(), "csky") {} +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/CSKYAttributes.h b/llvm/include/llvm/Support/CSKYAttributes.h new file mode 100644 index 000000000000..723f2ceee8fb --- /dev/null +++ b/llvm/include/llvm/Support/CSKYAttributes.h @@ -0,0 +1,95 @@ +//===---- CSKYAttributes.h - CSKY Attributes --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains enumerations for CSKY attributes. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_SUPPORT_CSKYATTRIBUTES_H +#define LLVM_SUPPORT_CSKYATTRIBUTES_H + +#include "llvm/Support/ELFAttributes.h" + +namespace llvm { +namespace CSKYAttrs { + +const TagNameMap &getCSKYAttributeTags(); + +enum AttrType { + CSKY_ARCH_NAME = 4, + CSKY_CPU_NAME = 5, + CSKY_ISA_FLAGS = 6, + CSKY_ISA_EXT_FLAGS = 7, + CSKY_DSP_VERSION = 8, + CSKY_VDSP_VERSION = 9, + CSKY_FPU_VERSION = 16, + CSKY_FPU_ABI = 17, + CSKY_FPU_ROUNDING = 18, + CSKY_FPU_DENORMAL = 19, + CSKY_FPU_EXCEPTION = 20, + CSKY_FPU_NUMBER_MODULE = 21, + CSKY_FPU_HARDFP = 22 +}; + +enum ISA_FLAGS { + V2_ISA_E1 = 1 << 1, + V2_ISA_1E2 = 1 << 2, + V2_ISA_2E3 = 1 << 3, + V2_ISA_3E7 = 1 << 4, + V2_ISA_7E10 = 1 << 5, + V2_ISA_3E3R1 = 1 << 6, + V2_ISA_3E3R2 = 1 << 7, + V2_ISA_10E60 = 1 << 8, + V2_ISA_3E3R3 = 1 << 9, + ISA_TRUST = 1 << 11, + ISA_CACHE = 1 << 12, + ISA_NVIC = 1 << 13, + ISA_CP = 1 << 14, + ISA_MP = 1 << 15, + ISA_MP_1E2 = 1 << 16, + ISA_JAVA = 1 << 17, + ISA_MAC = 1 << 18, + ISA_MAC_DSP = 1 << 19, + ISA_DSP = 1 << 20, + ISA_DSP_1E2 = 1 << 21, + ISA_DSP_ENHANCE = 1 << 22, + ISA_DSP_SILAN = 1 << 23, + ISA_VDSP = 1 << 24, + ISA_VDSP_2 = 1 << 25, + ISA_VDSP_2E3 = 1 << 26, + V2_ISA_DSPE60 = 1 << 27, + ISA_VDSP_2E60F = 1 << 28 +}; + +enum ISA_EXT_FLAGS { + ISA_FLOAT_E1 = 1 << 0, + ISA_FLOAT_1E2 = 1 << 1, + ISA_FLOAT_1E3 = 1 << 2, + ISA_FLOAT_3E4 = 1 << 3, + ISA_FLOAT_7E60 = 1 << 4 +}; + +enum { NONE = 0, NEEDED = 1 }; + +enum DSP_VERSION { DSP_VERSION_EXTENSION = 1, DSP_VERSION_2 = 2 }; + +enum VDSP_VERSION { VDSP_VERSION_1 = 1, VDSP_VERSION_2 = 2 }; + +enum FPU_VERSION { FPU_VERSION_1 = 1, FPU_VERSION_2 = 2, FPU_VERSION_3 = 3 }; + +enum FPU_ABI { FPU_ABI_SOFT = 1, FPU_ABI_SOFTFP = 2, FPU_ABI_HARD = 3 }; + +enum FPU_HARDFP { + FPU_HARDFP_HALF = 1, + FPU_HARDFP_SINGLE = 2, + FPU_HARDFP_DOUBLE = 4 +}; + +} // namespace CSKYAttrs +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/CSKYTargetParser.def b/llvm/include/llvm/Support/CSKYTargetParser.def new file mode 100644 index 000000000000..c93d6fdf8cce --- /dev/null +++ b/llvm/include/llvm/Support/CSKYTargetParser.def @@ -0,0 +1,524 @@ +//===- CSKYTargetParser.def - CSKY target parsing defines -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides defines to build up the CSKY target parser's logic. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef CSKY_FPU +#define CSKY_FPU(NAME, KIND, VERSION) +#endif +CSKY_FPU("invalid", FK_INVALID, FPUVersion::NONE) +CSKY_FPU("auto", FK_AUTO, FPUVersion::FPV2) +CSKY_FPU("fpv2", FK_FPV2, FPUVersion::FPV2) +CSKY_FPU("fpv2_divd", FK_FPV2_DIVD, FPUVersion::FPV2) +CSKY_FPU("fpv2_sf", FK_FPV2_SF, FPUVersion::FPV2) +CSKY_FPU("fpv3", FK_FPV3, FPUVersion::FPV3) +CSKY_FPU("fpv3_hf", FK_FPV3_HF, FPUVersion::FPV3) +CSKY_FPU("fpv3_hsf", FK_FPV3_HSF, FPUVersion::FPV3) +CSKY_FPU("fpv3_sdf", FK_FPV3_SDF, FPUVersion::FPV3) + +#undef CSKY_FPU + +#ifndef CSKY_ARCH +#define CSKY_ARCH(NAME, ID, ARCH_BASE_EXT) +#endif +CSKY_ARCH("invalid", INVALID, CSKY::AEK_INVALID) +CSKY_ARCH("ck801", CK801, CSKY::MAEK_E1 | CSKY::AEK_TRUST) +CSKY_ARCH("ck802", CK802, CSKY::MAEK_E2 | CSKY::AEK_TRUST | CSKY::AEK_NVIC) +CSKY_ARCH("ck803", CK803, + CSKY::MAEK_2E3 | CSKY::AEK_MP | CSKY::AEK_TRUST | CSKY::AEK_NVIC | + CSKY::AEK_HWDIV) +CSKY_ARCH("ck803s", CK803S, + CSKY::MAEK_2E3 | CSKY::AEK_MP | CSKY::AEK_TRUST | CSKY::AEK_NVIC | + CSKY::AEK_HWDIV) +CSKY_ARCH("ck804", CK804, + CSKY::MAEK_2E3 | CSKY::AEK_MP | CSKY::AEK_TRUST | CSKY::AEK_NVIC | + CSKY::AEK_HWDIV | CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) +CSKY_ARCH("ck805", CK805, + CSKY::MAEK_2E3 | CSKY::AEK_MP | CSKY::AEK_TRUST | CSKY::AEK_NVIC | + CSKY::AEK_HWDIV | CSKY::AEK_HIGHREG | CSKY::MAEK_3E3R2 | + CSKY::AEK_3E3R3 | CSKY::AEK_VDSPV2 | CSKY::AEK_VDSP2E3) +CSKY_ARCH("ck807", CK807, + CSKY::MAEK_3E7 | CSKY::MAEK_MP | CSKY::MAEK_MP1E2 | CSKY::AEK_TRUST | + CSKY::AEK_HWDIV | CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | + CSKY::AEK_DSPE60 | CSKY::AEK_HIGHREG | CSKY::AEK_HARDTP | + CSKY::AEK_NVIC | CSKY::AEK_CACHE) +CSKY_ARCH("ck810", CK810, + CSKY::MAEK_7E10 | CSKY::MAEK_MP | CSKY::MAEK_MP1E2 | CSKY::AEK_TRUST | + CSKY::AEK_HWDIV | CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | + CSKY::AEK_DSPE60 | CSKY::AEK_HIGHREG | CSKY::AEK_HARDTP | + CSKY::AEK_NVIC | CSKY::AEK_CACHE) +CSKY_ARCH("ck810v", CK810V, + CSKY::MAEK_7E10 | CSKY::MAEK_MP | CSKY::MAEK_MP1E2 | CSKY::AEK_TRUST | + CSKY::AEK_HWDIV | CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | + CSKY::AEK_DSPE60 | CSKY::AEK_HIGHREG | CSKY::AEK_HARDTP | + CSKY::AEK_NVIC | CSKY::AEK_CACHE | CSKY::AEK_VDSPV1) +CSKY_ARCH("ck860", CK860, + CSKY::MAEK_10E60 | CSKY::MAEK_MP | CSKY::MAEK_MP1E2 | + CSKY::AEK_TRUST | CSKY::AEK_HWDIV | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG | CSKY::AEK_HARDTP | CSKY::AEK_NVIC | + CSKY::AEK_CACHE | CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) +CSKY_ARCH("ck860v", CK860V, + CSKY::MAEK_10E60 | CSKY::MAEK_MP | CSKY::MAEK_MP1E2 | + CSKY::AEK_TRUST | CSKY::AEK_HWDIV | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG | CSKY::AEK_HARDTP | CSKY::AEK_NVIC | + CSKY::AEK_CACHE | CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | + CSKY::AEK_VDSPV2 | CSKY::AEK_VDSP2E60F) +#undef CSKY_ARCH + +#ifndef CSKY_ARCH_EXT_NAME +#define CSKY_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) +#endif +CSKY_ARCH_EXT_NAME("invalid", CSKY::AEK_INVALID, nullptr, nullptr) +CSKY_ARCH_EXT_NAME("none", CSKY::AEK_NONE, nullptr, nullptr) +CSKY_ARCH_EXT_NAME("fpuv2_sf", CSKY::AEK_FPUV2SF, "+fpuv2_sf", "-fpuv2_sf") +CSKY_ARCH_EXT_NAME("fpuv2_df", CSKY::AEK_FPUV2DF, "+fpuv2_df", "-fpuv2_df") +CSKY_ARCH_EXT_NAME("fdivdu", CSKY::AEK_FDIVDU, "+fdivdu", "-fdivdu") +CSKY_ARCH_EXT_NAME("fpuv3_hi", CSKY::AEK_FPUV3HI, "+fpuv3_hi", "-fpuv3_hi") +CSKY_ARCH_EXT_NAME("fpuv3_hf", CSKY::AEK_FPUV3HF, "+fpuv3_hf", "-fpuv3_hf") +CSKY_ARCH_EXT_NAME("fpuv3_sf", CSKY::AEK_FPUV3SF, "+fpuv3_sf", "-fpuv3_sf") +CSKY_ARCH_EXT_NAME("fpuv3_df", CSKY::AEK_FPUV3DF, "+fpuv3_df", "-fpuv3_df") +CSKY_ARCH_EXT_NAME("floate1", CSKY::AEK_FLOATE1, "+floate1", "-floate1") +CSKY_ARCH_EXT_NAME("float1e2", CSKY::AEK_FLOAT1E2, "+float1e2", "-float1e2") +CSKY_ARCH_EXT_NAME("float1e3", CSKY::AEK_FLOAT1E3, "+float1e3", "-float1e3") +CSKY_ARCH_EXT_NAME("float3e4", CSKY::AEK_FLOAT3E4, "+float3e4", "-float3e4") +CSKY_ARCH_EXT_NAME("float7e60", CSKY::AEK_FLOAT7E60, "+float7e60", "-float7e60") +CSKY_ARCH_EXT_NAME("hwdiv", CSKY::AEK_HWDIV, "+hwdiv", "-hwdiv") +CSKY_ARCH_EXT_NAME("multiple_stld", CSKY::AEK_STLD, "+multiple_stld", + "-multiple_stld") +CSKY_ARCH_EXT_NAME("pushpop", CSKY::AEK_PUSHPOP, "+pushpop", "-pushpop") +CSKY_ARCH_EXT_NAME("edsp", CSKY::AEK_EDSP, "+edsp", "-edsp") +CSKY_ARCH_EXT_NAME("dsp1e2", CSKY::AEK_DSP1E2, "+dsp1e2", "-dsp1e2") +CSKY_ARCH_EXT_NAME("dspe60", CSKY::AEK_DSPE60, "+dspe60", "-dspe60") +CSKY_ARCH_EXT_NAME("dspv2", CSKY::AEK_DSPV2, "+dspv2", "-dspv2") +CSKY_ARCH_EXT_NAME("dsp_silan", CSKY::AEK_DSPSILAN, "+dsp_silan", "-dsp_silan") +CSKY_ARCH_EXT_NAME("elrw", CSKY::AEK_ELRW, "+elrw", "-elrw") +CSKY_ARCH_EXT_NAME("trust", CSKY::AEK_TRUST, "+trust", "-trust") +CSKY_ARCH_EXT_NAME("java", CSKY::AEK_JAVA, "+java", "-java") +CSKY_ARCH_EXT_NAME("cache", CSKY::AEK_CACHE, "+cache", "-cache") +CSKY_ARCH_EXT_NAME("nvic", CSKY::AEK_NVIC, "+nvic", "-nvic") +CSKY_ARCH_EXT_NAME("doloop", CSKY::AEK_DOLOOP, "+doloop", "-doloop") +CSKY_ARCH_EXT_NAME("high-registers", CSKY::AEK_HIGHREG, "+high-registers", + "-high-registers") +CSKY_ARCH_EXT_NAME("smart", CSKY::AEK_SMART, "+smart", "-smart") +CSKY_ARCH_EXT_NAME("vdsp2e3", CSKY::AEK_VDSP2E3, "+vdsp2e3", "-vdsp2e3") +CSKY_ARCH_EXT_NAME("vdsp2e60f", CSKY::AEK_VDSP2E60F, "+vdsp2e60f", "-vdsp2e60f") +CSKY_ARCH_EXT_NAME("vdspv2", CSKY::AEK_VDSPV2, "+vdspv2", "-vdspv2") +CSKY_ARCH_EXT_NAME("hard-tp", CSKY::AEK_HARDTP, "+hard-tp", "-hard-tp") +CSKY_ARCH_EXT_NAME("soft-tp", CSKY::AEK_SOFTTP, "+soft-tp", "-soft-tp") +CSKY_ARCH_EXT_NAME("istack", CSKY::AEK_ISTACK, "+istack", "-istack") +CSKY_ARCH_EXT_NAME("constpool", CSKY::AEK_CONSTPOOL, "+constpool", "-constpool") +CSKY_ARCH_EXT_NAME("stack-size", CSKY::AEK_STACKSIZE, "+stack-size", + "-stack-size") +CSKY_ARCH_EXT_NAME("ccrt", CSKY::AEK_CCRT, "+ccrt", "-ccrt") +CSKY_ARCH_EXT_NAME("vdspv1", CSKY::AEK_VDSPV1, "+vdspv1", "-vdspv1") + +CSKY_ARCH_EXT_NAME("e1", CSKY::AEK_E1, "+e1", "-e1") +CSKY_ARCH_EXT_NAME("e2", CSKY::AEK_E2, "+e2", "-e2") +CSKY_ARCH_EXT_NAME("2e3", CSKY::AEK_2E3, "+2e3", "-2e3") +CSKY_ARCH_EXT_NAME("mp", CSKY::AEK_MP, "+mp", "-mp") +CSKY_ARCH_EXT_NAME("3e3r1", CSKY::AEK_3E3R1, "+3e3r1", "-3e3r1") +CSKY_ARCH_EXT_NAME("3e3r2", CSKY::AEK_3E3R2, "+3e3r2", "-3e3r2") +CSKY_ARCH_EXT_NAME("3e3r3", CSKY::AEK_3E3R3, "+3e3r3", "-3e3r3") +CSKY_ARCH_EXT_NAME("3e7", CSKY::AEK_3E7, "+3e7", "-3e7") +CSKY_ARCH_EXT_NAME("mp1e2", CSKY::AEK_MP1E2, "+mp1e2", "-mp1e2") +CSKY_ARCH_EXT_NAME("7e10", CSKY::AEK_7E10, "+7e10", "-7e10") +CSKY_ARCH_EXT_NAME("10e60", CSKY::AEK_10E60, "+10e60", "-10e60") + +#undef CSKY_ARCH_EXT_NAME + +#ifndef CSKY_CPU_NAME +#define CSKY_CPU_NAME(NAME, ARCH_ID, DEFAULT_EXT) +#endif + +CSKY_CPU_NAME("ck801", CK801, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck801t", CK801, CSKY::AEK_NONE) +CSKY_CPU_NAME("e801", CK801, CSKY::AEK_NONE) + +CSKY_CPU_NAME("ck802", CK802, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck802t", CK802, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck802j", CK802, CSKY::AEK_JAVA) +CSKY_CPU_NAME("e802", CK802, CSKY::AEK_NONE) +CSKY_CPU_NAME("e802t", CK802, CSKY::AEK_NONE) +CSKY_CPU_NAME("s802", CK802, CSKY::AEK_NONE) +CSKY_CPU_NAME("s802t", CK802, CSKY::AEK_NONE) + +CSKY_CPU_NAME("ck803", CK803, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803h", CK803, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803t", CK803, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803ht", CK803, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803f", CK803, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fh", CK803, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803e", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck803eh", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck803et", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck803eht", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck803ef", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803efh", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803ft", CK803, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803eft", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803efht", CK803, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803r1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803r2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803r3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803hr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803hr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803hr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803tr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803tr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803tr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803htr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803htr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803htr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2) +CSKY_CPU_NAME("ck803fr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fhr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fhr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803fhr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803er1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803er2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803er3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803etr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803etr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803etr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehtr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehtr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ehtr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803ftr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803ftr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803ftr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803eftr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803eftr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803eftr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhtr1", CK803, + CSKY::MAEK_3E3R1 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhtr2", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck803efhtr3", CK803, + CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3 | CSKY::AEK_DSPV2 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("s803", CK803, CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("s803t", CK803, CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("e803", CK803, CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("e803t", CK803, CSKY::MAEK_3E3R2 | CSKY::AEK_3E3R3) + +CSKY_CPU_NAME("ck803s", CK803S, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803st", CK803S, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck803se", CK803S, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck803sf", CK803S, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803sef", CK803S, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck803seft", CK803S, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) + +CSKY_CPU_NAME("ck804", CK804, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck804h", CK804, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck804t", CK804, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck804ht", CK804, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck804f", CK804, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck804fh", CK804, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck804e", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804eh", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804et", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804eht", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804ef", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804efh", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804ft", CK804, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck804eft", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("ck804efht", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("e804d", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("e804dt", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("e804f", CK804, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("e804ft", CK804, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("e804df", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) +CSKY_CPU_NAME("e804dft", CK804, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_HIGHREG) + +CSKY_CPU_NAME("ck805", CK805, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck805e", CK805, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("ck805f", CK805, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck805t", CK805, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck805ef", CK805, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck805et", CK805, + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("ck805ft", CK805, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) +CSKY_CPU_NAME("ck805eft", CK805, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_DSPV2 | CSKY::AEK_3E3R1 | CSKY::AEK_3E3R3) +CSKY_CPU_NAME("i805", CK805, CSKY::AEK_NONE) +CSKY_CPU_NAME("i805f", CK805, + CSKY::AEK_FPUV2SF | CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E3) + +CSKY_CPU_NAME("ck807", CK807, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck807e", CK807, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60) +CSKY_CPU_NAME("ck807f", CK807, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_FLOAT3E4) +CSKY_CPU_NAME("ck807ef", CK807, + CSKY::AEK_EDSP | CSKY::AEK_DSP1E2 | CSKY::AEK_DSPE60 | + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_FLOAT3E4) +CSKY_CPU_NAME("c807", CK807, CSKY::AEK_NONE) +CSKY_CPU_NAME("c807f", CK807, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_FLOAT3E4) +CSKY_CPU_NAME("r807", CK807, CSKY::AEK_NONE) +CSKY_CPU_NAME("r807f", CK807, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2 | CSKY::AEK_FLOAT1E3 | + CSKY::AEK_FLOAT3E4) + +CSKY_CPU_NAME("ck810e", CK810, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810et", CK810, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810ef", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810eft", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810", CK810, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810f", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810t", CK810, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810ft", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("c810", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("c810t", CK810, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) + +CSKY_CPU_NAME("ck810v", CK810V, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810ev", CK810V, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810tv", CK810V, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck810etv", CK810V, CSKY::AEK_NONE) +CSKY_CPU_NAME("c810v", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810fv", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810efv", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("ck810ftv", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("c810tv", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) +CSKY_CPU_NAME("c810eftv", CK810V, + CSKY::AEK_FPUV2SF | CSKY::AEK_FPUV2DF | CSKY::AEK_FDIVDU | + CSKY::AEK_FLOATE1 | CSKY::AEK_FLOAT1E2) + +CSKY_CPU_NAME("ck860", CK860, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck860f", CK860, + CSKY::AEK_FPUV3HI | CSKY::AEK_FPUV3HF | CSKY::AEK_FPUV3SF | + CSKY::AEK_FPUV3DF | CSKY::AEK_FLOAT7E60) +CSKY_CPU_NAME("c860", CK860, + CSKY::AEK_FPUV3HI | CSKY::AEK_FPUV3HF | CSKY::AEK_FPUV3SF | + CSKY::AEK_FPUV3DF | CSKY::AEK_FLOAT7E60) + +CSKY_CPU_NAME("ck860v", CK860V, CSKY::AEK_NONE) +CSKY_CPU_NAME("ck860fv", CK860V, + CSKY::AEK_FPUV3HI | CSKY::AEK_FPUV3HF | CSKY::AEK_FPUV3SF | + CSKY::AEK_FPUV3DF | CSKY::AEK_FLOAT7E60) +CSKY_CPU_NAME("c860v", CK860V, + CSKY::AEK_FPUV3HI | CSKY::AEK_FPUV3HF | CSKY::AEK_FPUV3SF | + CSKY::AEK_FPUV3DF | CSKY::AEK_FLOAT7E60) +// Invalid CPU +CSKY_CPU_NAME("invalid", INVALID, CSKY::AEK_INVALID) +#undef CSKY_CPU_NAME diff --git a/llvm/include/llvm/Support/CSKYTargetParser.h b/llvm/include/llvm/Support/CSKYTargetParser.h new file mode 100644 index 000000000000..ca33a7ee406c --- /dev/null +++ b/llvm/include/llvm/Support/CSKYTargetParser.h @@ -0,0 +1,203 @@ +//===-- CSKYTargetParser - Parser for CSKY target features --------*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a target parser to recognise CSKY hardware features +// such as FPU/CPU/ARCH/extensions and specific support such as HWDIV. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CSKYTARGETPARSER_H +#define LLVM_SUPPORT_CSKYTARGETPARSER_H + +#include "llvm/ADT/Triple.h" +#include <vector> + +namespace llvm { +class StringRef; + +namespace CSKY { + +// Arch extension modifiers for CPUs. +enum ArchExtKind : uint64_t { + AEK_INVALID = 0, + AEK_NONE = 1, + AEK_FPUV2SF = 1 << 1, + AEK_FPUV2DF = 1 << 2, + AEK_FDIVDU = 1 << 3, + AEK_FPUV3HI = 1 << 4, + AEK_FPUV3HF = 1 << 5, + AEK_FPUV3SF = 1 << 6, + AEK_FPUV3DF = 1 << 7, + AEK_FLOATE1 = 1 << 8, + AEK_FLOAT1E2 = 1 << 9, + AEK_FLOAT1E3 = 1 << 10, + AEK_FLOAT3E4 = 1 << 11, + AEK_FLOAT7E60 = 1 << 12, + AEK_HWDIV = 1 << 13, + AEK_STLD = 1 << 14, + AEK_PUSHPOP = 1 << 15, + AEK_EDSP = 1 << 16, + AEK_DSP1E2 = 1 << 17, + AEK_DSPE60 = 1 << 18, + AEK_DSPV2 = 1 << 19, + AEK_DSPSILAN = 1 << 20, + AEK_ELRW = 1 << 21, + AEK_TRUST = 1 << 22, + AEK_JAVA = 1 << 23, + AEK_CACHE = 1 << 24, + AEK_NVIC = 1 << 25, + AEK_DOLOOP = 1 << 26, + AEK_HIGHREG = 1 << 27, + AEK_SMART = 1 << 28, + AEK_VDSP2E3 = 1 << 29, + AEK_VDSP2E60F = 1 << 30, + AEK_VDSPV2 = 1ULL << 31, + AEK_HARDTP = 1ULL << 32, + AEK_SOFTTP = 1ULL << 33, + AEK_ISTACK = 1ULL << 34, + AEK_CONSTPOOL = 1ULL << 35, + AEK_STACKSIZE = 1ULL << 36, + AEK_CCRT = 1ULL << 37, + AEK_VDSPV1 = 1ULL << 38, + AEK_E1 = 1ULL << 39, + AEK_E2 = 1ULL << 40, + AEK_2E3 = 1ULL << 41, + AEK_MP = 1ULL << 42, + AEK_3E3R1 = 1ULL << 43, + AEK_3E3R2 = 1ULL << 44, + AEK_3E3R3 = 1ULL << 45, + AEK_3E7 = 1ULL << 46, + AEK_MP1E2 = 1ULL << 47, + AEK_7E10 = 1ULL << 48, + AEK_10E60 = 1ULL << 49 + +}; + +// Arch extension modifiers for CPUs. +enum MultiArchExtKind : uint64_t { + MAEK_E1 = CSKY::AEK_E1 | CSKY::AEK_ELRW, + MAEK_E2 = CSKY::AEK_E2 | CSKY::MAEK_E1, + MAEK_2E3 = CSKY::AEK_2E3 | CSKY::MAEK_E2, + MAEK_MP = CSKY::AEK_MP | CSKY::MAEK_2E3, + MAEK_3E3R1 = CSKY::AEK_3E3R1, + MAEK_3E3R2 = CSKY::AEK_3E3R1 | CSKY::AEK_3E3R2 | CSKY::AEK_DOLOOP, + MAEK_3E7 = CSKY::AEK_3E7 | CSKY::MAEK_2E3, + MAEK_MP1E2 = CSKY::AEK_MP1E2 | CSKY::MAEK_3E7, + MAEK_7E10 = CSKY::AEK_7E10 | CSKY::MAEK_3E7, + MAEK_10E60 = CSKY::AEK_10E60 | CSKY::MAEK_7E10, +}; +// FPU names. +enum CSKYFPUKind { +#define CSKY_FPU(NAME, KIND, VERSION) KIND, +#include "CSKYTargetParser.def" + FK_LAST +}; + +// FPU Version +enum class FPUVersion { + NONE, + FPV2, + FPV3, +}; + +// Arch names. +enum class ArchKind { +#define CSKY_ARCH(NAME, ID, ARCH_BASE_EXT) ID, +#include "CSKYTargetParser.def" +}; + +// List of Arch Extension names. +// FIXME: TableGen this. +struct ExtName { + const char *NameCStr; + size_t NameLength; + uint64_t ID; + const char *Feature; + const char *NegFeature; + + StringRef getName() const { return StringRef(NameCStr, NameLength); } +}; + +const CSKY::ExtName CSKYARCHExtNames[] = { +#define CSKY_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) \ + {NAME, sizeof(NAME) - 1, ID, FEATURE, NEGFEATURE}, +#include "CSKYTargetParser.def" +}; + +// List of CPU names and their arches. +template <typename T> struct CpuNames { + const char *NameCStr; + size_t NameLength; + T ArchID; + uint64_t defaultExt; + + StringRef getName() const { return StringRef(NameCStr, NameLength); } +}; +const CpuNames<CSKY::ArchKind> CPUNames[] = { +#define CSKY_CPU_NAME(NAME, ARCH_ID, DEFAULT_EXT) \ + {NAME, sizeof(NAME) - 1, CSKY::ArchKind::ARCH_ID, DEFAULT_EXT}, +#include "llvm/Support/CSKYTargetParser.def" +}; + +// FIXME: TableGen this. +// The entries must appear in the order listed in CSKY::CSKYFPUKind for correct +// indexing +struct FPUName { + const char *NameCStr; + size_t NameLength; + CSKYFPUKind ID; + FPUVersion FPUVer; + + StringRef getName() const { return StringRef(NameCStr, NameLength); } +}; + +static const FPUName FPUNames[] = { +#define CSKY_FPU(NAME, KIND, VERSION) {NAME, sizeof(NAME) - 1, KIND, VERSION}, +#include "llvm/Support/CSKYTargetParser.def" +}; + +// List of canonical arch names. +template <typename T> struct ArchNames { + const char *NameCStr; + size_t NameLength; + T ID; + uint64_t archBaseExt; + StringRef getName() const { return StringRef(NameCStr, NameLength); } +}; +const ArchNames<CSKY::ArchKind> ARCHNames[] = { +#define CSKY_ARCH(NAME, ID, ARCH_BASE_EXT) \ + {NAME, sizeof(NAME) - 1, CSKY::ArchKind::ID, ARCH_BASE_EXT}, +#include "llvm/Support/CSKYTargetParser.def" +}; + +StringRef getArchName(ArchKind AK); +StringRef getDefaultCPU(StringRef Arch); +StringRef getArchExtName(uint64_t ArchExtKind); +StringRef getArchExtFeature(StringRef ArchExt); +uint64_t getDefaultExtensions(StringRef CPU); +bool getExtensionFeatures(uint64_t Extensions, + std::vector<StringRef> &Features); + +// Information by ID +StringRef getFPUName(unsigned FPUKind); +FPUVersion getFPUVersion(unsigned FPUKind); + +bool getFPUFeatures(CSKYFPUKind Kind, std::vector<StringRef> &Features); + +// Parser +ArchKind parseArch(StringRef Arch); +ArchKind parseCPUArch(StringRef CPU); +uint64_t parseArchExt(StringRef ArchExt); +void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values); + +} // namespace CSKY + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/Casting.h b/llvm/include/llvm/Support/Casting.h index d6f7793d5df0..894c1f439b64 100644 --- a/llvm/include/llvm/Support/Casting.h +++ b/llvm/include/llvm/Support/Casting.h @@ -6,14 +6,15 @@ // //===----------------------------------------------------------------------===// // -// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(), -// and dyn_cast_or_null<X>() templates. +// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), +// cast_if_present<X>(), and dyn_cast_if_present<X>() templates. // //===----------------------------------------------------------------------===// #ifndef LLVM_SUPPORT_CASTING_H #define LLVM_SUPPORT_CASTING_H +#include "llvm/ADT/Optional.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/type_traits.h" #include <cassert> @@ -23,43 +24,47 @@ namespace llvm { //===----------------------------------------------------------------------===// -// isa<x> Support Templates +// simplify_type //===----------------------------------------------------------------------===// -// Define a template that can be specialized by smart pointers to reflect the -// fact that they are automatically dereferenced, and are not involved with the -// template selection process... the default implementation is a noop. -// -template<typename From> struct simplify_type { +/// Define a template that can be specialized by smart pointers to reflect the +/// fact that they are automatically dereferenced, and are not involved with the +/// template selection process... the default implementation is a noop. +// TODO: rename this and/or replace it with other cast traits. +template <typename From> struct simplify_type { using SimpleType = From; // The real type this represents... // An accessor to get the real value... static SimpleType &getSimplifiedValue(From &Val) { return Val; } }; -template<typename From> struct simplify_type<const From> { +template <typename From> struct simplify_type<const From> { using NonConstSimpleType = typename simplify_type<From>::SimpleType; - using SimpleType = - typename add_const_past_pointer<NonConstSimpleType>::type; + using SimpleType = typename add_const_past_pointer<NonConstSimpleType>::type; using RetType = typename add_lvalue_reference_if_not_pointer<SimpleType>::type; - static RetType getSimplifiedValue(const From& Val) { - return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val)); + static RetType getSimplifiedValue(const From &Val) { + return simplify_type<From>::getSimplifiedValue(const_cast<From &>(Val)); } }; +// TODO: add this namespace once everyone is switched to using the new +// interface. +// namespace detail { + +//===----------------------------------------------------------------------===// +// isa_impl +//===----------------------------------------------------------------------===// + // The core of the implementation of isa<X> is here; To and From should be // the names of classes. This template can be specialized to customize the // implementation of isa<> without rewriting it from scratch. -template <typename To, typename From, typename Enabler = void> -struct isa_impl { - static inline bool doit(const From &Val) { - return To::classof(&Val); - } +template <typename To, typename From, typename Enabler = void> struct isa_impl { + static inline bool doit(const From &Val) { return To::classof(&Val); } }; -/// Always allow upcasts, and perform no dynamic check for them. +// Always allow upcasts, and perform no dynamic check for them. template <typename To, typename From> struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> { static inline bool doit(const From &) { return true; } @@ -85,103 +90,78 @@ struct isa_impl_cl<To, const std::unique_ptr<From>> { } }; -template <typename To, typename From> struct isa_impl_cl<To, From*> { +template <typename To, typename From> struct isa_impl_cl<To, From *> { static inline bool doit(const From *Val) { assert(Val && "isa<> used on a null pointer"); return isa_impl<To, From>::doit(*Val); } }; -template <typename To, typename From> struct isa_impl_cl<To, From*const> { +template <typename To, typename From> struct isa_impl_cl<To, From *const> { static inline bool doit(const From *Val) { assert(Val && "isa<> used on a null pointer"); return isa_impl<To, From>::doit(*Val); } }; -template <typename To, typename From> struct isa_impl_cl<To, const From*> { +template <typename To, typename From> struct isa_impl_cl<To, const From *> { static inline bool doit(const From *Val) { assert(Val && "isa<> used on a null pointer"); return isa_impl<To, From>::doit(*Val); } }; -template <typename To, typename From> struct isa_impl_cl<To, const From*const> { +template <typename To, typename From> +struct isa_impl_cl<To, const From *const> { static inline bool doit(const From *Val) { assert(Val && "isa<> used on a null pointer"); return isa_impl<To, From>::doit(*Val); } }; -template<typename To, typename From, typename SimpleFrom> +template <typename To, typename From, typename SimpleFrom> struct isa_impl_wrap { // When From != SimplifiedType, we can simplify the type some more by using // the simplify_type template. static bool doit(const From &Val) { return isa_impl_wrap<To, SimpleFrom, - typename simplify_type<SimpleFrom>::SimpleType>::doit( - simplify_type<const From>::getSimplifiedValue(Val)); + typename simplify_type<SimpleFrom>::SimpleType>:: + doit(simplify_type<const From>::getSimplifiedValue(Val)); } }; -template<typename To, typename FromTy> +template <typename To, typename FromTy> struct isa_impl_wrap<To, FromTy, FromTy> { // When From == SimpleType, we are as simple as we are going to get. static bool doit(const FromTy &Val) { - return isa_impl_cl<To,FromTy>::doit(Val); + return isa_impl_cl<To, FromTy>::doit(Val); } }; -// isa<X> - Return true if the parameter to the template is an instance of one -// of the template type arguments. Used like this: -// -// if (isa<Type>(myVal)) { ... } -// if (isa<Type0, Type1, Type2>(myVal)) { ... } -// -template <class X, class Y> LLVM_NODISCARD inline bool isa(const Y &Val) { - return isa_impl_wrap<X, const Y, - typename simplify_type<const Y>::SimpleType>::doit(Val); -} - -template <typename First, typename Second, typename... Rest, typename Y> -LLVM_NODISCARD inline bool isa(const Y &Val) { - return isa<First>(Val) || isa<Second, Rest...>(Val); -} - -// isa_and_nonnull<X> - Functionally identical to isa, except that a null value -// is accepted. -// -template <typename... X, class Y> -LLVM_NODISCARD inline bool isa_and_nonnull(const Y &Val) { - if (!Val) - return false; - return isa<X...>(Val); -} - //===----------------------------------------------------------------------===// -// cast<x> Support Templates +// cast_retty + cast_retty_impl //===----------------------------------------------------------------------===// -template<class To, class From> struct cast_retty; +template <class To, class From> struct cast_retty; // Calculate what type the 'cast' function should return, based on a requested // type of To and a source type of From. -template<class To, class From> struct cast_retty_impl { - using ret_type = To &; // Normal case, return Ty& +template <class To, class From> struct cast_retty_impl { + using ret_type = To &; // Normal case, return Ty& }; -template<class To, class From> struct cast_retty_impl<To, const From> { +template <class To, class From> struct cast_retty_impl<To, const From> { using ret_type = const To &; // Normal case, return Ty& }; -template<class To, class From> struct cast_retty_impl<To, From*> { - using ret_type = To *; // Pointer arg case, return Ty* +template <class To, class From> struct cast_retty_impl<To, From *> { + using ret_type = To *; // Pointer arg case, return Ty* }; -template<class To, class From> struct cast_retty_impl<To, const From*> { +template <class To, class From> struct cast_retty_impl<To, const From *> { using ret_type = const To *; // Constant pointer arg case, return const Ty* }; -template<class To, class From> struct cast_retty_impl<To, const From*const> { +template <class To, class From> struct cast_retty_impl<To, const From *const> { using ret_type = const To *; // Constant pointer arg case, return const Ty* }; @@ -195,187 +175,604 @@ public: using ret_type = std::unique_ptr<ResultType>; }; -template<class To, class From, class SimpleFrom> -struct cast_retty_wrap { +template <class To, class From, class SimpleFrom> struct cast_retty_wrap { // When the simplified type and the from type are not the same, use the type // simplifier to reduce the type, then reuse cast_retty_impl to get the // resultant type. using ret_type = typename cast_retty<To, SimpleFrom>::ret_type; }; -template<class To, class FromTy> -struct cast_retty_wrap<To, FromTy, FromTy> { +template <class To, class FromTy> struct cast_retty_wrap<To, FromTy, FromTy> { // When the simplified type is equal to the from type, use it directly. - using ret_type = typename cast_retty_impl<To,FromTy>::ret_type; + using ret_type = typename cast_retty_impl<To, FromTy>::ret_type; }; -template<class To, class From> -struct cast_retty { +template <class To, class From> struct cast_retty { using ret_type = typename cast_retty_wrap< To, From, typename simplify_type<From>::SimpleType>::ret_type; }; +//===----------------------------------------------------------------------===// +// cast_convert_val +//===----------------------------------------------------------------------===// + // Ensure the non-simple values are converted using the simplify_type template // that may be specialized by smart pointers... // -template<class To, class From, class SimpleFrom> struct cast_convert_val { +template <class To, class From, class SimpleFrom> struct cast_convert_val { // This is not a simple type, use the template to simplify it... - static typename cast_retty<To, From>::ret_type doit(From &Val) { + static typename cast_retty<To, From>::ret_type doit(const From &Val) { return cast_convert_val<To, SimpleFrom, - typename simplify_type<SimpleFrom>::SimpleType>::doit( - simplify_type<From>::getSimplifiedValue(Val)); + typename simplify_type<SimpleFrom>::SimpleType>:: + doit(simplify_type<From>::getSimplifiedValue(const_cast<From &>(Val))); } }; -template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> { - // This _is_ a simple type, just cast it. +template <class To, class FromTy> struct cast_convert_val<To, FromTy, FromTy> { + // If it's a reference, switch to a pointer to do the cast and then deref it. static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) { - typename cast_retty<To, FromTy>::ret_type Res2 - = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val); - return Res2; + return *(std::remove_reference_t<typename cast_retty<To, FromTy>::ret_type> + *)&const_cast<FromTy &>(Val); + } +}; + +template <class To, class FromTy> +struct cast_convert_val<To, FromTy *, FromTy *> { + // If it's a pointer, we can use c-style casting directly. + static typename cast_retty<To, FromTy *>::ret_type doit(const FromTy *Val) { + return (typename cast_retty<To, FromTy *>::ret_type) const_cast<FromTy *>( + Val); } }; +//===----------------------------------------------------------------------===// +// is_simple_type +//===----------------------------------------------------------------------===// + template <class X> struct is_simple_type { static const bool value = std::is_same<X, typename simplify_type<X>::SimpleType>::value; }; -// cast<X> - Return the argument parameter cast to the specified type. This -// casting operator asserts that the type is correct, so it does not return null -// on failure. It does not allow a null argument (use cast_or_null for that). -// It is typically used like this: -// -// cast<Instruction>(myVal)->getParent() -// -template <class X, class Y> -inline std::enable_if_t<!is_simple_type<Y>::value, - typename cast_retty<X, const Y>::ret_type> -cast(const Y &Val) { - assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!"); - return cast_convert_val< - X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val); +// } // namespace detail + +//===----------------------------------------------------------------------===// +// CastIsPossible +//===----------------------------------------------------------------------===// + +/// This struct provides a way to check if a given cast is possible. It provides +/// a static function called isPossible that is used to check if a cast can be +/// performed. It should be overridden like this: +/// +/// template<> struct CastIsPossible<foo, bar> { +/// static inline bool isPossible(const bar &b) { +/// return bar.isFoo(); +/// } +/// }; +template <typename To, typename From, typename Enable = void> +struct CastIsPossible { + static inline bool isPossible(const From &f) { + return isa_impl_wrap< + To, const From, + typename simplify_type<const From>::SimpleType>::doit(f); + } +}; + +// Needed for optional unwrapping. This could be implemented with isa_impl, but +// we want to implement things in the new method and move old implementations +// over. In fact, some of the isa_impl templates should be moved over to +// CastIsPossible. +template <typename To, typename From> +struct CastIsPossible<To, Optional<From>> { + static inline bool isPossible(const Optional<From> &f) { + assert(f.hasValue() && "CastIsPossible::isPossible called on a nullopt!"); + return isa_impl_wrap< + To, const From, + typename simplify_type<const From>::SimpleType>::doit(*f); + } +}; + +/// Upcasting (from derived to base) and casting from a type to itself should +/// always be possible. +template <typename To, typename From> +struct CastIsPossible<To, From, + std::enable_if_t<std::is_base_of<To, From>::value>> { + static inline bool isPossible(const From &f) { return true; } +}; + +//===----------------------------------------------------------------------===// +// Cast traits +//===----------------------------------------------------------------------===// + +/// All of these cast traits are meant to be implementations for useful casts +/// that users may want to use that are outside the standard behavior. An +/// example of how to use a special cast called `CastTrait` is: +/// +/// template<> struct CastInfo<foo, bar> : public CastTrait<foo, bar> {}; +/// +/// Essentially, if your use case falls directly into one of the use cases +/// supported by a given cast trait, simply inherit your special CastInfo +/// directly from one of these to avoid having to reimplement the boilerplate +/// `isPossible/castFailed/doCast/doCastIfPossible`. A cast trait can also +/// provide a subset of those functions. + +/// This cast trait just provides castFailed for the specified `To` type to make +/// CastInfo specializations more declarative. In order to use this, the target +/// result type must be `To` and `To` must be constructible from `nullptr`. +template <typename To> struct NullableValueCastFailed { + static To castFailed() { return To(nullptr); } +}; + +/// This cast trait just provides the default implementation of doCastIfPossible +/// to make CastInfo specializations more declarative. The `Derived` template +/// parameter *must* be provided for forwarding castFailed and doCast. +template <typename To, typename From, typename Derived> +struct DefaultDoCastIfPossible { + static To doCastIfPossible(From f) { + if (!Derived::isPossible(f)) + return Derived::castFailed(); + return Derived::doCast(f); + } +}; + +namespace detail { +/// A helper to derive the type to use with `Self` for cast traits, when the +/// provided CRTP derived type is allowed to be void. +template <typename OptionalDerived, typename Default> +using SelfType = std::conditional_t<std::is_same<OptionalDerived, void>::value, + Default, OptionalDerived>; +} // namespace detail + +/// This cast trait provides casting for the specific case of casting to a +/// value-typed object from a pointer-typed object. Note that `To` must be +/// nullable/constructible from a pointer to `From` to use this cast. +template <typename To, typename From, typename Derived = void> +struct ValueFromPointerCast + : public CastIsPossible<To, From *>, + public NullableValueCastFailed<To>, + public DefaultDoCastIfPossible< + To, From *, + detail::SelfType<Derived, ValueFromPointerCast<To, From>>> { + static inline To doCast(From *f) { return To(f); } +}; + +/// This cast trait provides std::unique_ptr casting. It has the semantics of +/// moving the contents of the input unique_ptr into the output unique_ptr +/// during the cast. It's also a good example of how to implement a move-only +/// cast. +template <typename To, typename From, typename Derived = void> +struct UniquePtrCast : public CastIsPossible<To, From *> { + using Self = detail::SelfType<Derived, UniquePtrCast<To, From>>; + using CastResultType = std::unique_ptr< + std::remove_reference_t<typename cast_retty<To, From>::ret_type>>; + + static inline CastResultType doCast(std::unique_ptr<From> &&f) { + return CastResultType((typename CastResultType::element_type *)f.release()); + } + + static inline CastResultType castFailed() { return CastResultType(nullptr); } + + static inline CastResultType doCastIfPossible(std::unique_ptr<From> &&f) { + if (!Self::isPossible(f)) + return castFailed(); + return doCast(f); + } +}; + +/// This cast trait provides Optional<T> casting. This means that if you have a +/// value type, you can cast it to another value type and have dyn_cast return +/// an Optional<T>. +template <typename To, typename From, typename Derived = void> +struct OptionalValueCast + : public CastIsPossible<To, From>, + public DefaultDoCastIfPossible< + Optional<To>, From, + detail::SelfType<Derived, OptionalValueCast<To, From>>> { + static inline Optional<To> castFailed() { return Optional<To>{}; } + + static inline Optional<To> doCast(const From &f) { return To(f); } +}; + +/// Provides a cast trait that strips `const` from types to make it easier to +/// implement a const-version of a non-const cast. It just removes boilerplate +/// and reduces the amount of code you as the user need to implement. You can +/// use it like this: +/// +/// template<> struct CastInfo<foo, bar> { +/// ...verbose implementation... +/// }; +/// +/// template<> struct CastInfo<foo, const bar> : public +/// ConstStrippingForwardingCast<foo, const bar, CastInfo<foo, bar>> {}; +/// +template <typename To, typename From, typename ForwardTo> +struct ConstStrippingForwardingCast { + // Remove the pointer if it exists, then we can get rid of consts/volatiles. + using DecayedFrom = std::remove_cv_t<std::remove_pointer_t<From>>; + // Now if it's a pointer, add it back. Otherwise, we want a ref. + using NonConstFrom = std::conditional_t<std::is_pointer<From>::value, + DecayedFrom *, DecayedFrom &>; + + static inline bool isPossible(const From &f) { + return ForwardTo::isPossible(const_cast<NonConstFrom>(f)); + } + + static inline decltype(auto) castFailed() { return ForwardTo::castFailed(); } + + static inline decltype(auto) doCast(const From &f) { + return ForwardTo::doCast(const_cast<NonConstFrom>(f)); + } + + static inline decltype(auto) doCastIfPossible(const From &f) { + return ForwardTo::doCastIfPossible(const_cast<NonConstFrom>(f)); + } +}; + +/// Provides a cast trait that uses a defined pointer to pointer cast as a base +/// for reference-to-reference casts. Note that it does not provide castFailed +/// and doCastIfPossible because a pointer-to-pointer cast would likely just +/// return `nullptr` which could cause nullptr dereference. You can use it like +/// this: +/// +/// template <> struct CastInfo<foo, bar *> { ... verbose implementation... }; +/// +/// template <> +/// struct CastInfo<foo, bar> +/// : public ForwardToPointerCast<foo, bar, CastInfo<foo, bar *>> {}; +/// +template <typename To, typename From, typename ForwardTo> +struct ForwardToPointerCast { + static inline bool isPossible(const From &f) { + return ForwardTo::isPossible(&f); + } + + static inline decltype(auto) doCast(const From &f) { + return *ForwardTo::doCast(&f); + } +}; + +//===----------------------------------------------------------------------===// +// CastInfo +//===----------------------------------------------------------------------===// + +/// This struct provides a method for customizing the way a cast is performed. +/// It inherits from CastIsPossible, to support the case of declaring many +/// CastIsPossible specializations without having to specialize the full +/// CastInfo. +/// +/// In order to specialize different behaviors, specify different functions in +/// your CastInfo specialization. +/// For isa<> customization, provide: +/// +/// `static bool isPossible(const From &f)` +/// +/// For cast<> customization, provide: +/// +/// `static To doCast(const From &f)` +/// +/// For dyn_cast<> and the *_if_present<> variants' customization, provide: +/// +/// `static To castFailed()` and `static To doCastIfPossible(const From &f)` +/// +/// Your specialization might look something like this: +/// +/// template<> struct CastInfo<foo, bar> : public CastIsPossible<foo, bar> { +/// static inline foo doCast(const bar &b) { +/// return foo(const_cast<bar &>(b)); +/// } +/// static inline foo castFailed() { return foo(); } +/// static inline foo doCastIfPossible(const bar &b) { +/// if (!CastInfo<foo, bar>::isPossible(b)) +/// return castFailed(); +/// return doCast(b); +/// } +/// }; + +// The default implementations of CastInfo don't use cast traits for now because +// we need to specify types all over the place due to the current expected +// casting behavior and the way cast_retty works. New use cases can and should +// take advantage of the cast traits whenever possible! + +template <typename To, typename From, typename Enable = void> +struct CastInfo : public CastIsPossible<To, From> { + using Self = CastInfo<To, From, Enable>; + + using CastReturnType = typename cast_retty<To, From>::ret_type; + + static inline CastReturnType doCast(const From &f) { + return cast_convert_val< + To, From, + typename simplify_type<From>::SimpleType>::doit(const_cast<From &>(f)); + } + + // This assumes that you can construct the cast return type from `nullptr`. + // This is largely to support legacy use cases - if you don't want this + // behavior you should specialize CastInfo for your use case. + static inline CastReturnType castFailed() { return CastReturnType(nullptr); } + + static inline CastReturnType doCastIfPossible(const From &f) { + if (!Self::isPossible(f)) + return castFailed(); + return doCast(f); + } +}; + +/// This struct provides an overload for CastInfo where From has simplify_type +/// defined. This simply forwards to the appropriate CastInfo with the +/// simplified type/value, so you don't have to implement both. +template <typename To, typename From> +struct CastInfo<To, From, std::enable_if_t<!is_simple_type<From>::value>> { + using Self = CastInfo<To, From>; + using SimpleFrom = typename simplify_type<From>::SimpleType; + using SimplifiedSelf = CastInfo<To, SimpleFrom>; + + static inline bool isPossible(From &f) { + return SimplifiedSelf::isPossible( + simplify_type<From>::getSimplifiedValue(f)); + } + + static inline decltype(auto) doCast(From &f) { + return SimplifiedSelf::doCast(simplify_type<From>::getSimplifiedValue(f)); + } + + static inline decltype(auto) castFailed() { + return SimplifiedSelf::castFailed(); + } + + static inline decltype(auto) doCastIfPossible(From &f) { + return SimplifiedSelf::doCastIfPossible( + simplify_type<From>::getSimplifiedValue(f)); + } +}; + +//===----------------------------------------------------------------------===// +// Pre-specialized CastInfo +//===----------------------------------------------------------------------===// + +/// Provide a CastInfo specialized for std::unique_ptr. +template <typename To, typename From> +struct CastInfo<To, std::unique_ptr<From>> : public UniquePtrCast<To, From> {}; + +/// Provide a CastInfo specialized for Optional<From>. It's assumed that if the +/// input is Optional<From> that the output can be Optional<To>. If that's not +/// the case, specialize CastInfo for your use case. +template <typename To, typename From> +struct CastInfo<To, Optional<From>> : public OptionalValueCast<To, From> {}; + +/// isa<X> - Return true if the parameter to the template is an instance of one +/// of the template type arguments. Used like this: +/// +/// if (isa<Type>(myVal)) { ... } +/// if (isa<Type0, Type1, Type2>(myVal)) { ... } +template <typename To, typename From> +LLVM_NODISCARD inline bool isa(const From &Val) { + return CastInfo<To, const From>::isPossible(Val); } -template <class X, class Y> -inline typename cast_retty<X, Y>::ret_type cast(Y &Val) { - assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!"); - return cast_convert_val<X, Y, - typename simplify_type<Y>::SimpleType>::doit(Val); +template <typename First, typename Second, typename... Rest, typename From> +LLVM_NODISCARD inline bool isa(const From &Val) { + return isa<First>(Val) || isa<Second, Rest...>(Val); } -template <class X, class Y> -inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) { - assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!"); - return cast_convert_val<X, Y*, - typename simplify_type<Y*>::SimpleType>::doit(Val); +/// cast<X> - Return the argument parameter cast to the specified type. This +/// casting operator asserts that the type is correct, so it does not return +/// null on failure. It does not allow a null argument (use cast_if_present for +/// that). It is typically used like this: +/// +/// cast<Instruction>(myVal)->getParent() + +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) cast(const From &Val) { + assert(isa<To>(Val) && "cast<Ty>() argument of incompatible type!"); + return CastInfo<To, const From>::doCast(Val); } -template <class X, class Y> -inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type -cast(std::unique_ptr<Y> &&Val) { - assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!"); - using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type; - return ret_type( - cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit( - Val.release())); +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) cast(From &Val) { + assert(isa<To>(Val) && "cast<Ty>() argument of incompatible type!"); + return CastInfo<To, From>::doCast(Val); } -// cast_or_null<X> - Functionally identical to cast, except that a null value is -// accepted. -// -template <class X, class Y> -LLVM_NODISCARD inline std::enable_if_t< - !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type> -cast_or_null(const Y &Val) { - if (!Val) - return nullptr; - assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"); - return cast<X>(Val); +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) cast(From *Val) { + assert(isa<To>(Val) && "cast<Ty>() argument of incompatible type!"); + return CastInfo<To, From *>::doCast(Val); } -template <class X, class Y> -LLVM_NODISCARD inline std::enable_if_t<!is_simple_type<Y>::value, - typename cast_retty<X, Y>::ret_type> -cast_or_null(Y &Val) { - if (!Val) - return nullptr; - assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"); - return cast<X>(Val); +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) cast(std::unique_ptr<From> &&Val) { + assert(isa<To>(Val) && "cast<Ty>() argument of incompatible type!"); + return CastInfo<To, std::unique_ptr<From>>::doCast(std::move(Val)); } -template <class X, class Y> -LLVM_NODISCARD inline typename cast_retty<X, Y *>::ret_type -cast_or_null(Y *Val) { - if (!Val) return nullptr; - assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"); - return cast<X>(Val); +/// dyn_cast<X> - Return the argument parameter cast to the specified type. This +/// casting operator returns null if the argument is of the wrong type, so it +/// can be used to test for a type as well as cast if successful. The value +/// passed in must be present, if not, use dyn_cast_if_present. This should be +/// used in the context of an if statement like this: +/// +/// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... } + +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) dyn_cast(const From &Val) { + return CastInfo<To, const From>::doCastIfPossible(Val); } -template <class X, class Y> -inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type -cast_or_null(std::unique_ptr<Y> &&Val) { - if (!Val) - return nullptr; - return cast<X>(std::move(Val)); +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) dyn_cast(From &Val) { + return CastInfo<To, From>::doCastIfPossible(Val); } -// dyn_cast<X> - Return the argument parameter cast to the specified type. This -// casting operator returns null if the argument is of the wrong type, so it can -// be used to test for a type as well as cast if successful. This should be -// used in the context of an if statement like this: -// -// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... } -// +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) dyn_cast(From *Val) { + return CastInfo<To, From *>::doCastIfPossible(Val); +} -template <class X, class Y> -LLVM_NODISCARD inline std::enable_if_t< - !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type> -dyn_cast(const Y &Val) { - return isa<X>(Val) ? cast<X>(Val) : nullptr; +template <typename To, typename From> +LLVM_NODISCARD inline decltype(auto) dyn_cast(std::unique_ptr<From> &&Val) { + return CastInfo<To, std::unique_ptr<From>>::doCastIfPossible(std::move(Val)); } -template <class X, class Y> -LLVM_NODISCARD inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) { - return isa<X>(Val) ? cast<X>(Val) : nullptr; +//===----------------------------------------------------------------------===// +// ValueIsPresent +//===----------------------------------------------------------------------===// + +template <typename T> +constexpr bool IsNullable = std::is_pointer<T>::value || + std::is_constructible<T, std::nullptr_t>::value; + +/// ValueIsPresent provides a way to check if a value is, well, present. For +/// pointers, this is the equivalent of checking against nullptr, for +/// Optionals this is the equivalent of checking hasValue(). It also +/// provides a method for unwrapping a value (think dereferencing a +/// pointer). + +// Generic values can't *not* be present. +template <typename T, typename Enable = void> struct ValueIsPresent { + using UnwrappedType = T; + static inline bool isPresent(const T &t) { return true; } + static inline decltype(auto) unwrapValue(T &t) { return t; } +}; + +// Optional provides its own way to check if something is present. +template <typename T> struct ValueIsPresent<Optional<T>> { + using UnwrappedType = T; + static inline bool isPresent(const Optional<T> &t) { return t.has_value(); } + static inline decltype(auto) unwrapValue(Optional<T> &t) { + return t.getValue(); + } +}; + +// If something is "nullable" then we just compare it to nullptr to see if it +// exists. +template <typename T> +struct ValueIsPresent<T, std::enable_if_t<IsNullable<T>>> { + using UnwrappedType = T; + static inline bool isPresent(const T &t) { return t != nullptr; } + static inline decltype(auto) unwrapValue(T &t) { return t; } +}; + +namespace detail { +// Convenience function we can use to check if a value is present. Because of +// simplify_type, we have to call it on the simplified type for now. +template <typename T> inline bool isPresent(const T &t) { + return ValueIsPresent<typename simplify_type<T>::SimpleType>::isPresent( + simplify_type<T>::getSimplifiedValue(const_cast<T &>(t))); } -template <class X, class Y> -LLVM_NODISCARD inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) { - return isa<X>(Val) ? cast<X>(Val) : nullptr; +// Convenience function we can use to unwrap a value. +template <typename T> inline decltype(auto) unwrapValue(T &t) { + return ValueIsPresent<T>::unwrapValue(t); } +} // namespace detail -// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null -// value is accepted. -// -template <class X, class Y> -LLVM_NODISCARD inline std::enable_if_t< - !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type> -dyn_cast_or_null(const Y &Val) { - return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr; +/// isa_and_present<X> - Functionally identical to isa, except that a null value +/// is accepted. +template <typename... X, class Y> +LLVM_NODISCARD inline bool isa_and_present(const Y &Val) { + if (!detail::isPresent(Val)) + return false; + return isa<X...>(Val); } +template <typename... X, class Y> +LLVM_NODISCARD inline bool isa_and_nonnull(const Y &Val) { + return isa_and_present<X...>(Val); +} + +/// cast_if_present<X> - Functionally identical to cast, except that a null +/// value is accepted. template <class X, class Y> -LLVM_NODISCARD inline std::enable_if_t<!is_simple_type<Y>::value, - typename cast_retty<X, Y>::ret_type> -dyn_cast_or_null(Y &Val) { - return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr; +LLVM_NODISCARD inline auto cast_if_present(const Y &Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, const Y>::castFailed(); + assert(isa<X>(Val) && "cast_if_present<Ty>() argument of incompatible type!"); + return cast<X>(detail::unwrapValue(Val)); +} + +template <class X, class Y> LLVM_NODISCARD inline auto cast_if_present(Y &Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, Y>::castFailed(); + assert(isa<X>(Val) && "cast_if_present<Ty>() argument of incompatible type!"); + return cast<X>(detail::unwrapValue(Val)); +} + +template <class X, class Y> LLVM_NODISCARD inline auto cast_if_present(Y *Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, Y *>::castFailed(); + assert(isa<X>(Val) && "cast_if_present<Ty>() argument of incompatible type!"); + return cast<X>(detail::unwrapValue(Val)); } template <class X, class Y> -LLVM_NODISCARD inline typename cast_retty<X, Y *>::ret_type -dyn_cast_or_null(Y *Val) { - return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr; +LLVM_NODISCARD inline auto cast_if_present(std::unique_ptr<Y> &&Val) { + if (!detail::isPresent(Val)) + return UniquePtrCast<X, Y>::castFailed(); + return UniquePtrCast<X, Y>::doCast(std::move(Val)); +} + +// Provide a forwarding from cast_or_null to cast_if_present for current +// users. This is deprecated and will be removed in a future patch, use +// cast_if_present instead. +template <class X, class Y> auto cast_or_null(const Y &Val) { + return cast_if_present<X>(Val); +} + +template <class X, class Y> auto cast_or_null(Y &Val) { + return cast_if_present<X>(Val); +} + +template <class X, class Y> auto cast_or_null(Y *Val) { + return cast_if_present<X>(Val); +} + +template <class X, class Y> auto cast_or_null(std::unique_ptr<Y> &&Val) { + return cast_if_present<X>(std::move(Val)); +} + +/// dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a +/// null (or none in the case of optionals) value is accepted. +template <class X, class Y> auto dyn_cast_if_present(const Y &Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, const Y>::castFailed(); + return CastInfo<X, const Y>::doCastIfPossible(detail::unwrapValue(Val)); +} + +template <class X, class Y> auto dyn_cast_if_present(Y &Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, Y>::castFailed(); + return CastInfo<X, Y>::doCastIfPossible(detail::unwrapValue(Val)); +} + +template <class X, class Y> auto dyn_cast_if_present(Y *Val) { + if (!detail::isPresent(Val)) + return CastInfo<X, Y *>::castFailed(); + return CastInfo<X, Y *>::doCastIfPossible(detail::unwrapValue(Val)); +} + +// Forwards to dyn_cast_if_present to avoid breaking current users. This is +// deprecated and will be removed in a future patch, use +// cast_if_present instead. +template <class X, class Y> auto dyn_cast_or_null(const Y &Val) { + return dyn_cast_if_present<X>(Val); +} + +template <class X, class Y> auto dyn_cast_or_null(Y &Val) { + return dyn_cast_if_present<X>(Val); +} + +template <class X, class Y> auto dyn_cast_or_null(Y *Val) { + return dyn_cast_if_present<X>(Val); } -// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>, -// taking ownership of the input pointer iff isa<X>(Val) is true. If the -// cast is successful, From refers to nullptr on exit and the casted value -// is returned. If the cast is unsuccessful, the function returns nullptr -// and From is unchanged. +/// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>, +/// taking ownership of the input pointer iff isa<X>(Val) is true. If the +/// cast is successful, From refers to nullptr on exit and the casted value +/// is returned. If the cast is unsuccessful, the function returns nullptr +/// and From is unchanged. template <class X, class Y> -LLVM_NODISCARD inline auto unique_dyn_cast(std::unique_ptr<Y> &Val) - -> decltype(cast<X>(Val)) { +LLVM_NODISCARD inline typename CastInfo<X, std::unique_ptr<Y>>::CastResultType +unique_dyn_cast(std::unique_ptr<Y> &Val) { if (!isa<X>(Val)) return nullptr; return cast<X>(std::move(Val)); @@ -386,11 +783,11 @@ LLVM_NODISCARD inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) { return unique_dyn_cast<X, Y>(Val); } -// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that -// a null value is accepted. +// unique_dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, +// except that a null value is accepted. template <class X, class Y> -LLVM_NODISCARD inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val) - -> decltype(cast<X>(Val)) { +LLVM_NODISCARD inline typename CastInfo<X, std::unique_ptr<Y>>::CastResultType +unique_dyn_cast_or_null(std::unique_ptr<Y> &Val) { if (!Val) return nullptr; return unique_dyn_cast<X, Y>(Val); diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h index 9e66d84e185d..71d0ddbfe05e 100644 --- a/llvm/include/llvm/Support/CodeGen.h +++ b/llvm/include/llvm/Support/CodeGen.h @@ -69,6 +69,40 @@ namespace llvm { // Specify what functions should keep the frame pointer. enum class FramePointerKind { None, NonLeaf, All }; -} // end llvm namespace + // Specify what type of zeroing callee-used registers. + namespace ZeroCallUsedRegs { + const unsigned ONLY_USED = 1U << 1; + const unsigned ONLY_GPR = 1U << 2; + const unsigned ONLY_ARG = 1U << 3; + + enum class ZeroCallUsedRegsKind : unsigned int { + // Don't zero any call-used regs. + Skip = 1U << 0, + // Only zeros call-used GPRs used in the fn and pass args. + UsedGPRArg = ONLY_USED | ONLY_GPR | ONLY_ARG, + // Only zeros call-used GPRs used in the fn. + UsedGPR = ONLY_USED | ONLY_GPR, + // Only zeros call-used regs used in the fn and pass args. + UsedArg = ONLY_USED | ONLY_ARG, + // Only zeros call-used regs used in the fn. + Used = ONLY_USED, + // Zeros all call-used GPRs that pass args. + AllGPRArg = ONLY_GPR | ONLY_ARG, + // Zeros all call-used GPRs. + AllGPR = ONLY_GPR, + // Zeros all call-used regs that pass args. + AllArg = ONLY_ARG, + // Zeros all call-used regs. + All = 0, + }; + } // namespace ZeroCallUsedRegs + + enum class UWTableKind { + None = 0, ///< No unwind table requested + Sync = 1, ///< "Synchronous" unwind tables + Async = 2, ///< "Asynchronous" unwind tables (instr precise) + Default = 2, + }; + } // namespace llvm #endif diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h index c8e29ac42559..6461164fceff 100644 --- a/llvm/include/llvm/Support/CommandLine.h +++ b/llvm/include/llvm/Support/CommandLine.h @@ -49,13 +49,12 @@ class FileSystem; class StringSaver; -/// cl Namespace - This namespace contains all of the command line option -/// processing machinery. It is intentionally a short name to make qualified -/// usage concise. +/// This namespace contains all of the command line option processing machinery. +/// It is intentionally a short name to make qualified usage concise. namespace cl { //===----------------------------------------------------------------------===// -// ParseCommandLineOptions - Command line option processing entry point. +// Command line option processing entry point. // // Returns true on success. Otherwise, this will print the error message to // stderr and exit if \p Errs is not set (nullptr by default), or print the @@ -78,22 +77,19 @@ bool ParseCommandLineOptions(int argc, const char *const *argv, using VersionPrinterTy = std::function<void(raw_ostream &)>; ///===---------------------------------------------------------------------===// -/// SetVersionPrinter - Override the default (LLVM specific) version printer -/// used to print out the version when --version is given -/// on the command line. This allows other systems using the -/// CommandLine utilities to print their own version string. +/// Override the default (LLVM specific) version printer used to print out the +/// version when --version is given on the command line. This allows other +/// systems using the CommandLine utilities to print their own version string. void SetVersionPrinter(VersionPrinterTy func); ///===---------------------------------------------------------------------===// -/// AddExtraVersionPrinter - Add an extra printer to use in addition to the -/// default one. This can be called multiple times, -/// and each time it adds a new function to the list -/// which will be called after the basic LLVM version -/// printing is complete. Each can then add additional -/// information specific to the tool. +/// Add an extra printer to use in addition to the default one. This can be +/// called multiple times, and each time it adds a new function to the list +/// which will be called after the basic LLVM version printing is complete. +/// Each can then add additional information specific to the tool. void AddExtraVersionPrinter(VersionPrinterTy func); -// PrintOptionValues - Print option values. +// Print option values. // With -print-options print the difference between option values and defaults. // With -print-all-options print all option values. // (Currently not perfect, but best-effort.) @@ -121,9 +117,9 @@ enum NumOccurrencesFlag { // Flags for the number of occurrences allowed Required = 0x02, // One occurrence required OneOrMore = 0x03, // One or more occurrences required - // ConsumeAfter - Indicates that this option is fed anything that follows the - // last positional argument required by the application (it is an error if - // there are zero positional arguments, and a ConsumeAfter option is used). + // Indicates that this option is fed anything that follows the last positional + // argument required by the application (it is an error if there are zero + // positional arguments, and a ConsumeAfter option is used). // Thus, for example, all arguments to LLI are processed until a filename is // found. Once a filename is found, all of the succeeding arguments are // passed, unprocessed, to the ConsumeAfter option. @@ -144,8 +140,8 @@ enum OptionHidden { // Control whether -help shows this option ReallyHidden = 0x02 // Neither -help nor -help-hidden show this arg }; -// Formatting flags - This controls special features that the option might have -// that cause it to be parsed differently... +// This controls special features that the option might have that cause it to be +// parsed differently... // // Prefix - This option allows arguments that are otherwise unrecognized to be // matched by options that are a prefix of the actual value. This is useful for @@ -170,7 +166,7 @@ enum MiscFlags { // Miscellaneous flags to adjust argument PositionalEatsArgs = 0x02, // Should this positional cl::list eat -args? Sink = 0x04, // Should this cl::list eat all unknown options? - // Grouping - Can this option group with other options? + // Can this option group with other options? // If this is enabled, multiple letter options are allowed to bunch together // with only a single hyphen for the whole group. This allows emulation // of the behavior that ls uses for example: ls -la === ls -l -a @@ -181,7 +177,6 @@ enum MiscFlags { // Miscellaneous flags to adjust argument }; //===----------------------------------------------------------------------===// -// Option Category class // class OptionCategory { private: @@ -205,7 +200,6 @@ public: OptionCategory &getGeneralCategory(); //===----------------------------------------------------------------------===// -// SubCommand class // class SubCommand { private: @@ -244,14 +238,13 @@ extern ManagedStatic<SubCommand> TopLevelSubCommand; extern ManagedStatic<SubCommand> AllSubCommands; //===----------------------------------------------------------------------===// -// Option Base class // class Option { friend class alias; - // handleOccurrences - Overriden by subclasses to handle the value passed into - // an argument. Should return true if there was an error processing the - // argument and the program should exit. + // Overriden by subclasses to handle the value passed into an argument. Should + // return true if there was an error processing the argument and the program + // should exit. // virtual bool handleOccurrence(unsigned pos, StringRef ArgName, StringRef Arg) = 0; @@ -305,7 +298,7 @@ public: inline unsigned getPosition() const { return Position; } inline unsigned getNumAdditionalVals() const { return AdditionalVals; } - // hasArgStr - Return true if the argstr != "" + // Return true if the argstr != "" bool hasArgStr() const { return !ArgStr.empty(); } bool isPositional() const { return getFormattingFlag() == cl::Positional; } bool isSink() const { return getMiscFlags() & cl::Sink; } @@ -348,7 +341,7 @@ protected: public: virtual ~Option() = default; - // addArgument - Register this argument with the commandline system. + // Register this argument with the commandline system. // void addArgument(); @@ -361,8 +354,8 @@ public: // Return the width of the option tag for printing... virtual size_t getOptionWidth() const = 0; - // printOptionInfo - Print out information about this option. The - // to-be-maintained width is specified. + // Print out information about this option. The to-be-maintained width is + // specified. // virtual void printOptionInfo(size_t GlobalWidth) const = 0; @@ -388,7 +381,7 @@ public: virtual void getExtraOptionNames(SmallVectorImpl<StringRef> &) {} - // addOccurrence - Wrapper around handleOccurrence that enforces Flags. + // Wrapper around handleOccurrence that enforces Flags. // virtual bool addOccurrence(unsigned pos, StringRef ArgName, StringRef Value, bool MultiArg = false); @@ -408,7 +401,7 @@ public: // command line option parsers... // -// desc - Modifier to set the description shown in the -help output... +// Modifier to set the description shown in the -help output... struct desc { StringRef Desc; @@ -417,8 +410,7 @@ struct desc { void apply(Option &O) const { O.setDescription(Desc); } }; -// value_desc - Modifier to set the value description shown in the -help -// output... +// Modifier to set the value description shown in the -help output... struct value_desc { StringRef Desc; @@ -427,10 +419,9 @@ struct value_desc { void apply(Option &O) const { O.setValueStr(Desc); } }; -// init - Specify a default (initial) value for the command line argument, if -// the default constructor for the argument type does not give you what you -// want. This is only valid on "opt" arguments, not on "list" arguments. -// +// Specify a default (initial) value for the command line argument, if the +// default constructor for the argument type does not give you what you want. +// This is only valid on "opt" arguments, not on "list" arguments. template <class Ty> struct initializer { const Ty &Init; initializer(const Ty &Val) : Init(Val) {} @@ -442,10 +433,9 @@ template <class Ty> initializer<Ty> init(const Ty &Val) { return initializer<Ty>(Val); } -// location - Allow the user to specify which external variable they want to -// store the results of the command line argument processing into, if they don't -// want to store it in the option itself. -// +// Allow the user to specify which external variable they want to store the +// results of the command line argument processing into, if they don't want to +// store it in the option itself. template <class Ty> struct LocationClass { Ty &Loc; @@ -458,8 +448,7 @@ template <class Ty> LocationClass<Ty> location(Ty &L) { return LocationClass<Ty>(L); } -// cat - Specifiy the Option category for the command line argument to belong -// to. +// Specify the Option category for the command line argument to belong to. struct cat { OptionCategory &Category; @@ -468,7 +457,7 @@ struct cat { template <class Opt> void apply(Opt &O) const { O.addCategory(Category); } }; -// sub - Specify the subcommand that this option belongs to. +// Specify the subcommand that this option belongs to. struct sub { SubCommand ⋐ @@ -514,7 +503,6 @@ callback(F CB) { } //===----------------------------------------------------------------------===// -// OptionValue class // Support value comparison outside the template. struct GenericOptionValue { @@ -672,8 +660,8 @@ struct OptionEnumValue { #define clEnumValN(ENUMVAL, FLAGNAME, DESC) \ llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC } -// values - For custom data types, allow specifying a group of values together -// as the values that go into the mapping that the option handler uses. +// For custom data types, allow specifying a group of values together as the +// values that go into the mapping that the option handler uses. // class ValuesClass { // Use a vector instead of a map, because the lists should be short, @@ -699,16 +687,16 @@ template <typename... OptsTy> ValuesClass values(OptsTy... Options) { } //===----------------------------------------------------------------------===// -// parser class - Parameterizable parser for different data types. By default, -// known data types (string, int, bool) have specialized parsers, that do what -// you would expect. The default parser, used for data types that are not -// built-in, uses a mapping table to map specific options to values, which is -// used, among other things, to handle enum types. +// Parameterizable parser for different data types. By default, known data types +// (string, int, bool) have specialized parsers, that do what you would expect. +// The default parser, used for data types that are not built-in, uses a mapping +// table to map specific options to values, which is used, among other things, +// to handle enum types. //-------------------------------------------------- -// generic_parser_base - This class holds all the non-generic code that we do -// not need replicated for every instance of the generic parser. This also -// allows us to put stuff into CommandLine.cpp +// This class holds all the non-generic code that we do not need replicated for +// every instance of the generic parser. This also allows us to put stuff into +// CommandLine.cpp // class generic_parser_base { protected: @@ -726,15 +714,15 @@ public: virtual ~generic_parser_base() = default; // Base class should have virtual-destructor - // getNumOptions - Virtual function implemented by generic subclass to - // indicate how many entries are in Values. + // Virtual function implemented by generic subclass to indicate how many + // entries are in Values. // virtual unsigned getNumOptions() const = 0; - // getOption - Return option name N. + // Return option name N. virtual StringRef getOption(unsigned N) const = 0; - // getDescription - Return description N + // Return description N virtual StringRef getDescription(unsigned N) const = 0; // Return the width of the option tag for printing... @@ -742,8 +730,8 @@ public: virtual const GenericOptionValue &getOptionValue(unsigned N) const = 0; - // printOptionInfo - Print out information about this option. The - // to-be-maintained width is specified. + // Print out information about this option. The to-be-maintained width is + // specified. // virtual void printOptionInfo(const Option &O, size_t GlobalWidth) const; @@ -751,7 +739,7 @@ public: const GenericOptionValue &Default, size_t GlobalWidth) const; - // printOptionDiff - print the value of an option and it's default. + // Print the value of an option and it's default. // // Template definition ensures that the option and default have the same // DataType (via the same AnyOptionValue). @@ -791,7 +779,7 @@ public: return ValueDisallowed; } - // findOption - Return the option number corresponding to the specified + // Return the option number corresponding to the specified // argument string. If the option is not found, getNumOptions() is returned. // unsigned findOption(StringRef Name); @@ -829,12 +817,12 @@ public: return Values[N].HelpStr; } - // getOptionValue - Return the value of option name N. + // Return the value of option name N. const GenericOptionValue &getOptionValue(unsigned N) const override { return Values[N].V; } - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, DataType &V) { StringRef ArgVal; if (Owner.hasArgStr()) @@ -851,7 +839,7 @@ public: return O.error("Cannot find option named '" + ArgVal + "'!"); } - /// addLiteralOption - Add an entry to the mapping table. + /// Add an entry to the mapping table. /// template <class DT> void addLiteralOption(StringRef Name, const DT &V, StringRef HelpStr) { @@ -861,7 +849,7 @@ public: AddLiteralOption(Owner, Name); } - /// removeLiteralOption - Remove the specified option. + /// Remove the specified option. /// void removeLiteralOption(StringRef Name) { unsigned N = findOption(Name); @@ -871,7 +859,7 @@ public: }; //-------------------------------------------------- -// basic_parser - Super class of parsers to provide boilerplate code +// Super class of parsers to provide boilerplate code // class basic_parser_impl { // non-template implementation of basic_parser<t> public: @@ -890,16 +878,15 @@ public: // Return the width of the option tag for printing... size_t getOptionWidth(const Option &O) const; - // printOptionInfo - Print out information about this option. The - // to-be-maintained width is specified. + // Print out information about this option. The to-be-maintained width is + // specified. // void printOptionInfo(const Option &O, size_t GlobalWidth) const; - // printOptionNoValue - Print a placeholder for options that don't yet support - // printOptionDiff(). + // Print a placeholder for options that don't yet support printOptionDiff(). void printOptionNoValue(const Option &O, size_t GlobalWidth) const; - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. virtual StringRef getValueName() const { return "value"; } // An out-of-line virtual method to provide a 'home' for this class. @@ -910,8 +897,8 @@ protected: void printOptionName(const Option &O, size_t GlobalWidth) const; }; -// basic_parser - The real basic parser is just a template wrapper that provides -// a typedef for the provided data type. +// The real basic parser is just a template wrapper that provides a typedef for +// the provided data type. // template <class DataType> class basic_parser : public basic_parser_impl { public: @@ -922,8 +909,6 @@ public: }; //-------------------------------------------------- -// parser<bool> -// extern template class basic_parser<bool>; @@ -931,7 +916,7 @@ template <> class parser<bool> : public basic_parser<bool> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, bool &Val); void initialize() {} @@ -940,7 +925,7 @@ public: return ValueOptional; } - // getValueName - Do not print =<value> at all. + // Do not print =<value> at all. StringRef getValueName() const override { return StringRef(); } void printOptionDiff(const Option &O, bool V, OptVal Default, @@ -951,7 +936,6 @@ public: }; //-------------------------------------------------- -// parser<boolOrDefault> extern template class basic_parser<boolOrDefault>; @@ -959,14 +943,14 @@ template <> class parser<boolOrDefault> : public basic_parser<boolOrDefault> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, boolOrDefault &Val); enum ValueExpected getValueExpectedFlagDefault() const { return ValueOptional; } - // getValueName - Do not print =<value> at all. + // Do not print =<value> at all. StringRef getValueName() const override { return StringRef(); } void printOptionDiff(const Option &O, boolOrDefault V, OptVal Default, @@ -977,8 +961,6 @@ public: }; //-------------------------------------------------- -// parser<int> -// extern template class basic_parser<int>; @@ -986,10 +968,10 @@ template <> class parser<int> : public basic_parser<int> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, int &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "int"; } void printOptionDiff(const Option &O, int V, OptVal Default, @@ -1000,8 +982,6 @@ public: }; //-------------------------------------------------- -// parser<long> -// extern template class basic_parser<long>; @@ -1009,10 +989,10 @@ template <> class parser<long> final : public basic_parser<long> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, long &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "long"; } void printOptionDiff(const Option &O, long V, OptVal Default, @@ -1023,8 +1003,6 @@ public: }; //-------------------------------------------------- -// parser<long long> -// extern template class basic_parser<long long>; @@ -1032,10 +1010,10 @@ template <> class parser<long long> : public basic_parser<long long> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, long long &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "long"; } void printOptionDiff(const Option &O, long long V, OptVal Default, @@ -1046,8 +1024,6 @@ public: }; //-------------------------------------------------- -// parser<unsigned> -// extern template class basic_parser<unsigned>; @@ -1055,10 +1031,10 @@ template <> class parser<unsigned> : public basic_parser<unsigned> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, unsigned &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "uint"; } void printOptionDiff(const Option &O, unsigned V, OptVal Default, @@ -1069,8 +1045,6 @@ public: }; //-------------------------------------------------- -// parser<unsigned long> -// extern template class basic_parser<unsigned long>; @@ -1079,10 +1053,10 @@ class parser<unsigned long> final : public basic_parser<unsigned long> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, unsigned long &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "ulong"; } void printOptionDiff(const Option &O, unsigned long V, OptVal Default, @@ -1093,8 +1067,6 @@ public: }; //-------------------------------------------------- -// parser<unsigned long long> -// extern template class basic_parser<unsigned long long>; @@ -1103,11 +1075,11 @@ class parser<unsigned long long> : public basic_parser<unsigned long long> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, unsigned long long &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "ulong"; } void printOptionDiff(const Option &O, unsigned long long V, OptVal Default, @@ -1118,8 +1090,6 @@ public: }; //-------------------------------------------------- -// parser<double> -// extern template class basic_parser<double>; @@ -1127,10 +1097,10 @@ template <> class parser<double> : public basic_parser<double> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, double &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "number"; } void printOptionDiff(const Option &O, double V, OptVal Default, @@ -1141,8 +1111,6 @@ public: }; //-------------------------------------------------- -// parser<float> -// extern template class basic_parser<float>; @@ -1150,10 +1118,10 @@ template <> class parser<float> : public basic_parser<float> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, float &Val); - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "number"; } void printOptionDiff(const Option &O, float V, OptVal Default, @@ -1164,8 +1132,6 @@ public: }; //-------------------------------------------------- -// parser<std::string> -// extern template class basic_parser<std::string>; @@ -1173,13 +1139,13 @@ template <> class parser<std::string> : public basic_parser<std::string> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &, StringRef, StringRef Arg, std::string &Value) { Value = Arg.str(); return false; } - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "string"; } void printOptionDiff(const Option &O, StringRef V, const OptVal &Default, @@ -1190,8 +1156,6 @@ public: }; //-------------------------------------------------- -// parser<char> -// extern template class basic_parser<char>; @@ -1199,13 +1163,13 @@ template <> class parser<char> : public basic_parser<char> { public: parser(Option &O) : basic_parser(O) {} - // parse - Return true on error. + // Return true on error. bool parse(Option &, StringRef, StringRef Arg, char &Value) { Value = Arg[0]; return false; } - // getValueName - Overload in subclass to provide a better default value. + // Overload in subclass to provide a better default value. StringRef getValueName() const override { return "char"; } void printOptionDiff(const Option &O, char V, OptVal Default, @@ -1216,8 +1180,6 @@ public: }; //-------------------------------------------------- -// PrintOptionDiff -// // This collection of wrappers is the intermediary between class opt and class // parser to handle all the template nastiness. @@ -1261,10 +1223,10 @@ void printOptionDiff( } //===----------------------------------------------------------------------===// -// applicator class - This class is used because we must use partial -// specialization to handle literal string arguments specially (const char* does -// not correctly respond to the apply method). Because the syntax to use this -// is a pain, we have the 'apply' method below to handle the nastiness... +// This class is used because we must use partial specialization to handle +// literal string arguments specially (const char* does not correctly respond to +// the apply method). Because the syntax to use this is a pain, we have the +// 'apply' method below to handle the nastiness... // template <class Mod> struct applicator { template <class Opt> static void opt(const Mod &M, Opt &O) { M.apply(O); } @@ -1313,7 +1275,7 @@ template <> struct applicator<MiscFlags> { } }; -// apply method - Apply modifiers to an option in a type safe way. +// Apply modifiers to an option in a type safe way. template <class Opt, class Mod, class... Mods> void apply(Opt *O, const Mod &M, const Mods &... Ms) { applicator<Mod>::opt(M, *O); @@ -1325,8 +1287,6 @@ template <class Opt, class Mod> void apply(Opt *O, const Mod &M) { } //===----------------------------------------------------------------------===// -// opt_storage class - // Default storage class definition: external storage. This implementation // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. @@ -1406,7 +1366,7 @@ public: // Make sure we initialize the value with the default constructor for the // type. - opt_storage() : Value(DataType()), Default(DataType()) {} + opt_storage() : Value(DataType()), Default() {} template <class T> void setValue(const T &V, bool initial = false) { Value = V; @@ -1425,7 +1385,7 @@ public: }; //===----------------------------------------------------------------------===// -// opt - A scalar command line option. +// A scalar command line option. // template <class DataType, bool ExternalStorage = false, class ParserClass = parser<DataType>> @@ -1476,6 +1436,8 @@ class opt : public Option, const OptionValue<DataType> &V = this->getDefault(); if (V.hasValue()) this->setValue(V.getValue()); + else + this->setValue(T()); } template <class T, @@ -1528,8 +1490,6 @@ extern template class opt<char>; extern template class opt<bool>; //===----------------------------------------------------------------------===// -// list_storage class - // Default storage class definition: external storage. This implementation // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. @@ -1634,7 +1594,7 @@ public: }; //===----------------------------------------------------------------------===// -// list - A list of command line options. +// A list of command line options. // template <class DataType, class StorageClass = bool, class ParserClass = parser<DataType>> @@ -1716,7 +1676,7 @@ public: [](const typename ParserClass::parser_data_type &) {}; }; -// multi_val - Modifier to set the number of additional values. +// Modifier to set the number of additional values. struct multi_val { unsigned AdditionalVals; explicit multi_val(unsigned N) : AdditionalVals(N) {} @@ -1728,8 +1688,6 @@ struct multi_val { }; //===----------------------------------------------------------------------===// -// bits_storage class - // Default storage class definition: external storage. This implementation // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. @@ -1738,7 +1696,7 @@ template <class DataType, class StorageClass> class bits_storage { unsigned *Location = nullptr; // Where to store the bits... template <class T> static unsigned Bit(const T &V) { - unsigned BitPos = reinterpret_cast<unsigned>(V); + unsigned BitPos = static_cast<unsigned>(V); assert(BitPos < sizeof(unsigned) * CHAR_BIT && "enum exceeds width of bit vector!"); return 1 << BitPos; @@ -1763,6 +1721,11 @@ public: unsigned getBits() { return *Location; } + void clear() { + if (Location) + *Location = 0; + } + template <class T> bool isSet(const T &V) { return (*Location & Bit(V)) != 0; } @@ -1772,10 +1735,10 @@ public: // This makes us exactly compatible with the bits in all cases that it is used. // template <class DataType> class bits_storage<DataType, bool> { - unsigned Bits; // Where to store the bits... + unsigned Bits{0}; // Where to store the bits... template <class T> static unsigned Bit(const T &V) { - unsigned BitPos = (unsigned)V; + unsigned BitPos = static_cast<unsigned>(V); assert(BitPos < sizeof(unsigned) * CHAR_BIT && "enum exceeds width of bit vector!"); return 1 << BitPos; @@ -1786,11 +1749,13 @@ public: unsigned getBits() { return Bits; } + void clear() { Bits = 0; } + template <class T> bool isSet(const T &V) { return (Bits & Bit(V)) != 0; } }; //===----------------------------------------------------------------------===// -// bits - A bit vector of command options. +// A bit vector of command options. // template <class DataType, class Storage = bool, class ParserClass = parser<DataType>> @@ -1832,7 +1797,7 @@ class bits : public Option, public bits_storage<DataType, Storage> { void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { } - void setDefault() override {} + void setDefault() override { bits_storage<DataType, Storage>::clear(); } void done() { addArgument(); @@ -1929,7 +1894,7 @@ public: } }; -// aliasfor - Modifier to set the option an alias aliases. +// Modifier to set the option an alias aliases. struct aliasopt { Option &Opt; @@ -1938,10 +1903,9 @@ struct aliasopt { void apply(alias &A) const { A.setAliasFor(Opt); } }; -// extrahelp - provide additional help at the end of the normal help -// output. All occurrences of cl::extrahelp will be accumulated and -// printed to stderr at the end of the regular help, just before -// exit is called. +// Provide additional help at the end of the normal help output. All occurrences +// of cl::extrahelp will be accumulated and printed to stderr at the end of the +// regular help, just before exit is called. struct extrahelp { StringRef morehelp; @@ -2032,12 +1996,15 @@ void TokenizeGNUCommandLine(StringRef Source, StringSaver &Saver, SmallVectorImpl<const char *> &NewArgv, bool MarkEOLs = false); -/// Tokenizes a Windows command line which may contain quotes and escaped -/// quotes. +/// Tokenizes a string of Windows command line arguments, which may contain +/// quotes and escaped quotes. /// /// See MSDN docs for CommandLineToArgvW for information on the quoting rules. /// http://msdn.microsoft.com/en-us/library/windows/desktop/17w5ykft(v=vs.85).aspx /// +/// For handling a full Windows command line including the executable name at +/// the start, see TokenizeWindowsCommandLineFull below. +/// /// \param [in] Source The string to be split on whitespace with quotes. /// \param [in] Saver Delegates back to the caller for saving parsed strings. /// \param [in] MarkEOLs true if tokenizing a response file and you want end of @@ -2054,6 +2021,23 @@ void TokenizeWindowsCommandLine(StringRef Source, StringSaver &Saver, void TokenizeWindowsCommandLineNoCopy(StringRef Source, StringSaver &Saver, SmallVectorImpl<StringRef> &NewArgv); +/// Tokenizes a Windows full command line, including command name at the start. +/// +/// This uses the same syntax rules as TokenizeWindowsCommandLine for all but +/// the first token. But the first token is expected to be parsed as the +/// executable file name in the way CreateProcess would do it, rather than the +/// way the C library startup code would do it: CreateProcess does not consider +/// that \ is ever an escape character (because " is not a valid filename char, +/// hence there's never a need to escape it to be used literally). +/// +/// Parameters are the same as for TokenizeWindowsCommandLine. In particular, +/// if you set MarkEOLs = true, then the first word of every line will be +/// parsed using the special rules for command names, making this function +/// suitable for parsing a file full of commands to execute. +void TokenizeWindowsCommandLineFull(StringRef Source, StringSaver &Saver, + SmallVectorImpl<const char *> &NewArgv, + bool MarkEOLs = false); + /// String tokenization function type. Should be compatible with either /// Windows or Unix command line tokenizers. using TokenizerCallback = void (*)(StringRef Source, StringSaver &Saver, diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h index f3317049524f..6708b7cc95cc 100644 --- a/llvm/include/llvm/Support/Compiler.h +++ b/llvm/include/llvm/Support/Compiler.h @@ -39,6 +39,10 @@ # define __has_builtin(x) 0 #endif +#ifndef __has_include +# define __has_include(x) 0 +#endif + // Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in // C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid. #ifndef LLVM_HAS_CPP_ATTRIBUTE @@ -90,30 +94,14 @@ #define LLVM_MSC_PREREQ(version) (_MSC_VER >= (version)) // We require at least VS 2019. +#if !defined(LLVM_FORCE_USE_OLD_TOOLCHAIN) #if !LLVM_MSC_PREREQ(1920) #error LLVM requires at least VS 2019. #endif - -#else -#define LLVM_MSC_PREREQ(version) 0 #endif -/// Does the compiler support ref-qualifiers for *this? -/// -/// Sadly, this is separate from just rvalue reference support because GCC -/// and MSVC implemented this later than everything else. This appears to be -/// corrected in MSVC 2019 but not MSVC 2017. -/// FIXME: Remove LLVM_HAS_RVALUE_REFERENCE_THIS macro -#define LLVM_HAS_RVALUE_REFERENCE_THIS 1 - -/// Expands to '&' if ref-qualifiers for *this are supported. -/// -/// This can be used to provide lvalue/rvalue overrides of member functions. -/// The rvalue override should be guarded by LLVM_HAS_RVALUE_REFERENCE_THIS -#if LLVM_HAS_RVALUE_REFERENCE_THIS -#define LLVM_LVALUE_FUNCTION & #else -#define LLVM_LVALUE_FUNCTION +#define LLVM_MSC_PREREQ(version) 0 #endif /// LLVM_LIBRARY_VISIBILITY - If a class marked with this attribute is linked @@ -325,20 +313,17 @@ #define LLVM_EXTENSION #endif -// LLVM_ATTRIBUTE_DEPRECATED(decl, "message") -// This macro will be removed. -// Use C++14's attribute instead: [[deprecated("message")]] -#define LLVM_ATTRIBUTE_DEPRECATED(decl, message) [[deprecated(message)]] decl - /// LLVM_BUILTIN_UNREACHABLE - On compilers which support it, expands /// to an expression which states that it is undefined behavior for the /// compiler to reach this point. Otherwise is not defined. +/// +/// '#else' is intentionally left out so that other macro logic (e.g., +/// LLVM_ASSUME_ALIGNED and llvm_unreachable()) can detect whether +/// LLVM_BUILTIN_UNREACHABLE has a definition. #if __has_builtin(__builtin_unreachable) || defined(__GNUC__) # define LLVM_BUILTIN_UNREACHABLE __builtin_unreachable() #elif defined(_MSC_VER) # define LLVM_BUILTIN_UNREACHABLE __assume(false) -#else -# define LLVM_BUILTIN_UNREACHABLE #endif /// LLVM_BUILTIN_TRAP - On compilers which support it, expands to an expression @@ -411,22 +396,6 @@ # define LLVM_PACKED_END _Pragma("pack(pop)") #endif -/// \macro LLVM_PTR_SIZE -/// A constant integer equivalent to the value of sizeof(void*). -/// Generally used in combination with alignas or when doing computation in the -/// preprocessor. -#ifdef __SIZEOF_POINTER__ -# define LLVM_PTR_SIZE __SIZEOF_POINTER__ -#elif defined(_WIN64) -# define LLVM_PTR_SIZE 8 -#elif defined(_WIN32) -# define LLVM_PTR_SIZE 4 -#elif defined(_MSC_VER) -# error "could not determine LLVM_PTR_SIZE as a constant int for MSVC" -#else -# define LLVM_PTR_SIZE sizeof(void *) -#endif - /// \macro LLVM_MEMORY_SANITIZER_BUILD /// Whether LLVM itself is built with MemorySanitizer instrumentation. #if __has_feature(memory_sanitizer) @@ -444,8 +413,21 @@ /// Whether LLVM itself is built with AddressSanitizer instrumentation. #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) # define LLVM_ADDRESS_SANITIZER_BUILD 1 +#if __has_include(<sanitizer/asan_interface.h>) # include <sanitizer/asan_interface.h> #else +// These declarations exist to support ASan with MSVC. If MSVC eventually ships +// asan_interface.h in their headers, then we can remove this. +#ifdef __cplusplus +extern "C" { +#endif +void __asan_poison_memory_region(void const volatile *addr, size_t size); +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#ifdef __cplusplus +} // extern "C" +#endif +#endif +#else # define LLVM_ADDRESS_SANITIZER_BUILD 0 # define __asan_poison_memory_region(p, size) # define __asan_unpoison_memory_region(p, size) diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index 5bc0e56913fe..e6f898229412 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -29,8 +29,8 @@ static constexpr int BestSizeCompression = 9; bool isAvailable(); -Error compress(StringRef InputBuffer, SmallVectorImpl<char> &CompressedBuffer, - int Level = DefaultCompression); +void compress(StringRef InputBuffer, SmallVectorImpl<char> &CompressedBuffer, + int Level = DefaultCompression); Error uncompress(StringRef InputBuffer, char *UncompressedBuffer, size_t &UncompressedSize); diff --git a/llvm/include/llvm/Support/ConvertUTF.h b/llvm/include/llvm/Support/ConvertUTF.h index 374cdb907fdc..662f3aca5b54 100644 --- a/llvm/include/llvm/Support/ConvertUTF.h +++ b/llvm/include/llvm/Support/ConvertUTF.h @@ -126,6 +126,9 @@ typedef unsigned char Boolean; /* 0 or 1 */ #define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF #define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE +#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF +#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0xFFFE0000 + typedef enum { conversionOK, /* conversion successful */ sourceExhausted, /* partial character in source, but hit end */ @@ -282,6 +285,24 @@ bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out); bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out); /** + * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string. + * + * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out); + +/** + * Converts a UTF32 string into a UTF8 std::string. + * + * \param [in] Src A buffer of UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out); + +/** * Converts a UTF-8 string into a UTF-16 string with native endianness. * * \returns true on success diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h index f60e7335e197..26ddf97b3ef0 100644 --- a/llvm/include/llvm/Support/CrashRecoveryContext.h +++ b/llvm/include/llvm/Support/CrashRecoveryContext.h @@ -101,6 +101,9 @@ public: /// return failure from RunSafely(). This function does not return. [[noreturn]] void HandleExit(int RetCode); + /// Return true if RetCode indicates that a signal or an exception occurred. + static bool isCrash(int RetCode); + /// Throw again a signal or an exception, after it was catched once by a /// CrashRecoveryContext. static bool throwIfCrash(int RetCode); diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h index 2ff978476c79..5788ab3b2138 100644 --- a/llvm/include/llvm/Support/Debug.h +++ b/llvm/include/llvm/Support/Debug.h @@ -67,8 +67,8 @@ void setCurrentDebugTypes(const char **Types, unsigned Count); #else #define isCurrentDebugType(X) (false) -#define setCurrentDebugType(X) -#define setCurrentDebugTypes(X, N) +#define setCurrentDebugType(X) do { (void)(X); } while (false) +#define setCurrentDebugTypes(X, N) do { (void)(X); (void)(N); } while (false) #define DEBUG_WITH_TYPE(TYPE, X) do { } while (false) #endif diff --git a/llvm/include/llvm/Support/Errno.h b/llvm/include/llvm/Support/Errno.h index 07df6765d9db..e095c66b9086 100644 --- a/llvm/include/llvm/Support/Errno.h +++ b/llvm/include/llvm/Support/Errno.h @@ -15,7 +15,6 @@ #include <cerrno> #include <string> -#include <type_traits> namespace llvm { namespace sys { diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index 881049b15b0d..1a801b6f2c7a 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -1269,7 +1269,7 @@ public: void log(raw_ostream &OS) const override { assert(Err && "Trying to log after takeError()."); OS << "'" << FileName << "': "; - if (Line.hasValue()) + if (Line) OS << "line " << Line.getValue() << ": "; Err->log(OS); } @@ -1281,7 +1281,7 @@ public: return OS.str(); } - StringRef getFileName() { return FileName; } + StringRef getFileName() const { return FileName; } Error takeError() { return Error(std::move(Err)); } diff --git a/llvm/include/llvm/Support/ErrorHandling.h b/llvm/include/llvm/Support/ErrorHandling.h index f980510d37f0..004b3b7868fb 100644 --- a/llvm/include/llvm/Support/ErrorHandling.h +++ b/llvm/include/llvm/Support/ErrorHandling.h @@ -124,19 +124,30 @@ llvm_unreachable_internal(const char *msg = nullptr, const char *file = nullptr, /// Marks that the current location is not supposed to be reachable. /// In !NDEBUG builds, prints the message and location info to stderr. -/// In NDEBUG builds, becomes an optimizer hint that the current location -/// is not supposed to be reachable. On compilers that don't support -/// such hints, prints a reduced message instead and aborts the program. +/// In NDEBUG builds, if the platform does not support a builtin unreachable +/// then we call an internal LLVM runtime function. Otherwise the behavior is +/// controlled by the CMake flag +/// -DLLVM_UNREACHABLE_OPTIMIZE +/// * When "ON" (default) llvm_unreachable() becomes an optimizer hint +/// that the current location is not supposed to be reachable: the hint +/// turns such code path into undefined behavior. On compilers that don't +/// support such hints, prints a reduced message instead and aborts the +/// program. +/// * When "OFF", a builtin_trap is emitted instead of an +// optimizer hint or printing a reduced message. /// -/// Use this instead of assert(0). It conveys intent more clearly and -/// allows compilers to omit some unnecessary code. +/// Use this instead of assert(0). It conveys intent more clearly, suppresses +/// diagnostics for unreachable code paths, and allows compilers to omit +/// unnecessary code. #ifndef NDEBUG #define llvm_unreachable(msg) \ ::llvm::llvm_unreachable_internal(msg, __FILE__, __LINE__) -#elif defined(LLVM_BUILTIN_UNREACHABLE) +#elif !defined(LLVM_BUILTIN_UNREACHABLE) +#define llvm_unreachable(msg) ::llvm::llvm_unreachable_internal() +#elif LLVM_UNREACHABLE_OPTIMIZE #define llvm_unreachable(msg) LLVM_BUILTIN_UNREACHABLE #else -#define llvm_unreachable(msg) ::llvm::llvm_unreachable_internal() +#define llvm_unreachable(msg) LLVM_BUILTIN_TRAP, LLVM_BUILTIN_UNREACHABLE #endif #endif diff --git a/llvm/include/llvm/Support/FileUtilities.h b/llvm/include/llvm/Support/FileUtilities.h index f8a37fe1177d..0033638c6804 100644 --- a/llvm/include/llvm/Support/FileUtilities.h +++ b/llvm/include/llvm/Support/FileUtilities.h @@ -110,6 +110,27 @@ namespace llvm { llvm::Error writeFileAtomically(StringRef TempPathModel, StringRef FinalPath, std::function<llvm::Error(llvm::raw_ostream &)> Writer); + + /// FilePermssionsApplier helps to copy permissions from an input file to + /// an output one. It memorizes the status of the input file and can apply + /// permissions and dates to the output file. + class FilePermissionsApplier { + public: + static Expected<FilePermissionsApplier> create(StringRef InputFilename); + + /// Apply stored permissions to the \p OutputFilename. + /// Copy LastAccess and ModificationTime if \p CopyDates is true. + /// Overwrite stored permissions if \p OverwritePermissions is specified. + Error apply(StringRef OutputFilename, bool CopyDates = false, + Optional<sys::fs::perms> OverwritePermissions = None); + + private: + FilePermissionsApplier(StringRef InputFilename, sys::fs::file_status Status) + : InputFilename(InputFilename), InputStatus(Status) {} + + StringRef InputFilename; + sys::fs::file_status InputStatus; + }; } // End llvm namespace #endif diff --git a/llvm/include/llvm/Support/FormatProviders.h b/llvm/include/llvm/Support/FormatProviders.h index 3edd8844bc7a..8101ed7968ad 100644 --- a/llvm/include/llvm/Support/FormatProviders.h +++ b/llvm/include/llvm/Support/FormatProviders.h @@ -313,7 +313,7 @@ struct format_provider<T, S = FloatStyle::Fixed; Optional<size_t> Precision = parseNumericPrecision(Style); - if (!Precision.hasValue()) + if (!Precision) Precision = getDefaultPrecision(S); write_double(Stream, static_cast<double>(V), S, Precision); diff --git a/llvm/include/llvm/Support/FormatVariadic.h b/llvm/include/llvm/Support/FormatVariadic.h index a872afb5e45e..c1707b4fe9cb 100644 --- a/llvm/include/llvm/Support/FormatVariadic.h +++ b/llvm/include/llvm/Support/FormatVariadic.h @@ -172,7 +172,7 @@ public: // Formats textual output. `Fmt` is a string consisting of one or more // replacement sequences with the following grammar: // -// rep_field ::= "{" [index] ["," layout] [":" format] "}" +// rep_field ::= "{" index ["," layout] [":" format] "}" // index ::= <non-negative integer> // layout ::= [[[char]loc]width] // format ::= <any string not containing "{" or "}"> diff --git a/llvm/include/llvm/Support/HashBuilder.h b/llvm/include/llvm/Support/HashBuilder.h index bf93a0d22da7..9d7680d2b667 100644 --- a/llvm/include/llvm/Support/HashBuilder.h +++ b/llvm/include/llvm/Support/HashBuilder.h @@ -39,6 +39,9 @@ struct IsHashableData /// Declares the hasher member, and functions forwarding directly to the hasher. template <typename HasherT> class HashBuilderBase { public: + template <typename HasherT_ = HasherT> + using HashResultTy = decltype(std::declval<HasherT_ &>().final()); + HasherT &getHasher() { return Hasher; } /// Forward to `HasherT::update(ArrayRef<uint8_t>)`. @@ -59,12 +62,12 @@ public: } /// Forward to `HasherT::final()` if available. - template <typename HasherT_ = HasherT> StringRef final() { + template <typename HasherT_ = HasherT> HashResultTy<HasherT_> final() { return this->getHasher().final(); } /// Forward to `HasherT::result()` if available. - template <typename HasherT_ = HasherT> StringRef result() { + template <typename HasherT_ = HasherT> HashResultTy<HasherT_> result() { return this->getHasher().result(); } diff --git a/llvm/include/llvm/Support/Host.h b/llvm/include/llvm/Support/Host.h index b3c15f0683b9..f683371ad1d3 100644 --- a/llvm/include/llvm/Support/Host.h +++ b/llvm/include/llvm/Support/Host.h @@ -64,6 +64,7 @@ namespace sys { StringRef getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent); StringRef getHostCPUNameForARM(StringRef ProcCpuinfoContent); StringRef getHostCPUNameForS390x(StringRef ProcCpuinfoContent); + StringRef getHostCPUNameForRISCV(StringRef ProcCpuinfoContent); StringRef getHostCPUNameForBPF(); /// Helper functions to extract CPU details from CPUID on x86. diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index 96b7753e9b20..84e095e2bbab 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -324,7 +324,7 @@ public: /// Compute known bits resulting from multiplying LHS and RHS. static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, - bool SelfMultiply = false); + bool NoUndefSelfMultiply = false); /// Compute known bits from sign-extended multiply-hi. static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS); @@ -415,6 +415,12 @@ public: return KnownBits(Zero.reverseBits(), One.reverseBits()); } + bool operator==(const KnownBits &Other) const { + return Zero == Other.Zero && One == Other.One; + } + + bool operator!=(const KnownBits &Other) const { return !(*this == Other); } + void print(raw_ostream &OS) const; void dump() const; }; diff --git a/llvm/include/llvm/Support/LowLevelTypeImpl.h b/llvm/include/llvm/Support/LowLevelTypeImpl.h index dd286f5228fe..186a7e5930ec 100644 --- a/llvm/include/llvm/Support/LowLevelTypeImpl.h +++ b/llvm/include/llvm/Support/LowLevelTypeImpl.h @@ -207,6 +207,18 @@ public: return scalar(getScalarSizeInBits() / Factor); } + /// Produce a vector type that is \p Factor times bigger, preserving the + /// element type. For a scalar or pointer, this will produce a new vector with + /// \p Factor elements. + LLT multiplyElements(int Factor) const { + if (isVector()) { + return scalarOrVector(getElementCount().multiplyCoefficientBy(Factor), + getElementType()); + } + + return fixed_vector(Factor, *this); + } + bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); } unsigned getScalarSizeInBits() const { diff --git a/llvm/include/llvm/Support/MD5.h b/llvm/include/llvm/Support/MD5.h index 70d046601346..fa2f477261dd 100644 --- a/llvm/include/llvm/Support/MD5.h +++ b/llvm/include/llvm/Support/MD5.h @@ -40,26 +40,19 @@ template <typename T> class ArrayRef; class MD5 { public: - struct MD5Result { - std::array<uint8_t, 16> Bytes; - - operator std::array<uint8_t, 16>() const { return Bytes; } - - const uint8_t &operator[](size_t I) const { return Bytes[I]; } - uint8_t &operator[](size_t I) { return Bytes[I]; } - + struct MD5Result : public std::array<uint8_t, 16> { SmallString<32> digest() const; uint64_t low() const { // Our MD5 implementation returns the result in little endian, so the low // word is first. using namespace support; - return endian::read<uint64_t, little, unaligned>(Bytes.data()); + return endian::read<uint64_t, little, unaligned>(data()); } uint64_t high() const { using namespace support; - return endian::read<uint64_t, little, unaligned>(Bytes.data() + 8); + return endian::read<uint64_t, little, unaligned>(data() + 8); } std::pair<uint64_t, uint64_t> words() const { using namespace support; @@ -78,20 +71,20 @@ public: /// Finishes off the hash and puts the result in result. void final(MD5Result &Result); - /// Finishes off the hash, and returns a reference to the 16-byte hash data. - StringRef final(); + /// Finishes off the hash, and returns the 16-byte hash data. + MD5Result final(); - /// Finishes off the hash, and returns a reference to the 16-byte hash data. + /// Finishes off the hash, and returns the 16-byte hash data. /// This is suitable for getting the MD5 at any time without invalidating the /// internal state, so that more calls can be made into `update`. - StringRef result(); + MD5Result result(); /// Translates the bytes in \p Res to a hex string that is /// deposited into \p Str. The result will be of length 32. static void stringifyResult(MD5Result &Result, SmallVectorImpl<char> &Str); /// Computes the hash for a given bytes. - static std::array<uint8_t, 16> hash(ArrayRef<uint8_t> Data); + static MD5Result hash(ArrayRef<uint8_t> Data); private: // Any 32-bit or wider unsigned integer data type will do. @@ -109,15 +102,9 @@ private: MD5_u32plus block[16]; } InternalState; - MD5Result Result; - const uint8_t *body(ArrayRef<uint8_t> Data); }; -inline bool operator==(const MD5::MD5Result &LHS, const MD5::MD5Result &RHS) { - return LHS.Bytes == RHS.Bytes; -} - /// Helper to compute and return lower 64 bits of the given string's MD5 hash. inline uint64_t MD5Hash(StringRef Str) { using namespace support; diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h index 643c2d8ce981..5355c50bb762 100644 --- a/llvm/include/llvm/Support/MachineValueType.h +++ b/llvm/include/llvm/Support/MachineValueType.h @@ -41,143 +41,149 @@ namespace llvm { // ValueTypes.td as well! Other = 1, // This is a non-standard value i1 = 2, // This is a 1 bit integer value - i8 = 3, // This is an 8 bit integer value - i16 = 4, // This is a 16 bit integer value - i32 = 5, // This is a 32 bit integer value - i64 = 6, // This is a 64 bit integer value - i128 = 7, // This is a 128 bit integer value + i2 = 3, // This is a 2 bit integer value + i4 = 4, // This is a 4 bit integer value + i8 = 5, // This is an 8 bit integer value + i16 = 6, // This is a 16 bit integer value + i32 = 7, // This is a 32 bit integer value + i64 = 8, // This is a 64 bit integer value + i128 = 9, // This is a 128 bit integer value FIRST_INTEGER_VALUETYPE = i1, LAST_INTEGER_VALUETYPE = i128, - bf16 = 8, // This is a 16 bit brain floating point value - f16 = 9, // This is a 16 bit floating point value - f32 = 10, // This is a 32 bit floating point value - f64 = 11, // This is a 64 bit floating point value - f80 = 12, // This is a 80 bit floating point value - f128 = 13, // This is a 128 bit floating point value - ppcf128 = 14, // This is a PPC 128-bit floating point value + bf16 = 10, // This is a 16 bit brain floating point value + f16 = 11, // This is a 16 bit floating point value + f32 = 12, // This is a 32 bit floating point value + f64 = 13, // This is a 64 bit floating point value + f80 = 14, // This is a 80 bit floating point value + f128 = 15, // This is a 128 bit floating point value + ppcf128 = 16, // This is a PPC 128-bit floating point value FIRST_FP_VALUETYPE = bf16, LAST_FP_VALUETYPE = ppcf128, - v1i1 = 15, // 1 x i1 - v2i1 = 16, // 2 x i1 - v4i1 = 17, // 4 x i1 - v8i1 = 18, // 8 x i1 - v16i1 = 19, // 16 x i1 - v32i1 = 20, // 32 x i1 - v64i1 = 21, // 64 x i1 - v128i1 = 22, // 128 x i1 - v256i1 = 23, // 256 x i1 - v512i1 = 24, // 512 x i1 - v1024i1 = 25, // 1024 x i1 - - v1i8 = 26, // 1 x i8 - v2i8 = 27, // 2 x i8 - v4i8 = 28, // 4 x i8 - v8i8 = 29, // 8 x i8 - v16i8 = 30, // 16 x i8 - v32i8 = 31, // 32 x i8 - v64i8 = 32, // 64 x i8 - v128i8 = 33, // 128 x i8 - v256i8 = 34, // 256 x i8 - v512i8 = 35, // 512 x i8 - v1024i8 = 36, // 1024 x i8 - - v1i16 = 37, // 1 x i16 - v2i16 = 38, // 2 x i16 - v3i16 = 39, // 3 x i16 - v4i16 = 40, // 4 x i16 - v8i16 = 41, // 8 x i16 - v16i16 = 42, // 16 x i16 - v32i16 = 43, // 32 x i16 - v64i16 = 44, // 64 x i16 - v128i16 = 45, // 128 x i16 - v256i16 = 46, // 256 x i16 - v512i16 = 47, // 512 x i16 - - v1i32 = 48, // 1 x i32 - v2i32 = 49, // 2 x i32 - v3i32 = 50, // 3 x i32 - v4i32 = 51, // 4 x i32 - v5i32 = 52, // 5 x i32 - v6i32 = 53, // 6 x i32 - v7i32 = 54, // 7 x i32 - v8i32 = 55, // 8 x i32 - v16i32 = 56, // 16 x i32 - v32i32 = 57, // 32 x i32 - v64i32 = 58, // 64 x i32 - v128i32 = 59, // 128 x i32 - v256i32 = 60, // 256 x i32 - v512i32 = 61, // 512 x i32 - v1024i32 = 62, // 1024 x i32 - v2048i32 = 63, // 2048 x i32 - - v1i64 = 64, // 1 x i64 - v2i64 = 65, // 2 x i64 - v3i64 = 66, // 3 x i64 - v4i64 = 67, // 4 x i64 - v8i64 = 68, // 8 x i64 - v16i64 = 69, // 16 x i64 - v32i64 = 70, // 32 x i64 - v64i64 = 71, // 64 x i64 - v128i64 = 72, // 128 x i64 - v256i64 = 73, // 256 x i64 - - v1i128 = 74, // 1 x i128 + v1i1 = 17, // 1 x i1 + v2i1 = 18, // 2 x i1 + v4i1 = 19, // 4 x i1 + v8i1 = 20, // 8 x i1 + v16i1 = 21, // 16 x i1 + v32i1 = 22, // 32 x i1 + v64i1 = 23, // 64 x i1 + v128i1 = 24, // 128 x i1 + v256i1 = 25, // 256 x i1 + v512i1 = 26, // 512 x i1 + v1024i1 = 27, // 1024 x i1 + + v128i2 = 28, // 128 x i2 + + v64i4 = 29, // 64 x i4 + + v1i8 = 30, // 1 x i8 + v2i8 = 31, // 2 x i8 + v4i8 = 32, // 4 x i8 + v8i8 = 33, // 8 x i8 + v16i8 = 34, // 16 x i8 + v32i8 = 35, // 32 x i8 + v64i8 = 36, // 64 x i8 + v128i8 = 37, // 128 x i8 + v256i8 = 38, // 256 x i8 + v512i8 = 39, // 512 x i8 + v1024i8 = 40, // 1024 x i8 + + v1i16 = 41, // 1 x i16 + v2i16 = 42, // 2 x i16 + v3i16 = 43, // 3 x i16 + v4i16 = 44, // 4 x i16 + v8i16 = 45, // 8 x i16 + v16i16 = 46, // 16 x i16 + v32i16 = 47, // 32 x i16 + v64i16 = 48, // 64 x i16 + v128i16 = 49, // 128 x i16 + v256i16 = 50, // 256 x i16 + v512i16 = 51, // 512 x i16 + + v1i32 = 52, // 1 x i32 + v2i32 = 53, // 2 x i32 + v3i32 = 54, // 3 x i32 + v4i32 = 55, // 4 x i32 + v5i32 = 56, // 5 x i32 + v6i32 = 57, // 6 x i32 + v7i32 = 58, // 7 x i32 + v8i32 = 59, // 8 x i32 + v16i32 = 60, // 16 x i32 + v32i32 = 61, // 32 x i32 + v64i32 = 62, // 64 x i32 + v128i32 = 63, // 128 x i32 + v256i32 = 64, // 256 x i32 + v512i32 = 65, // 512 x i32 + v1024i32 = 66, // 1024 x i32 + v2048i32 = 67, // 2048 x i32 + + v1i64 = 68, // 1 x i64 + v2i64 = 69, // 2 x i64 + v3i64 = 70, // 3 x i64 + v4i64 = 71, // 4 x i64 + v8i64 = 72, // 8 x i64 + v16i64 = 73, // 16 x i64 + v32i64 = 74, // 32 x i64 + v64i64 = 75, // 64 x i64 + v128i64 = 76, // 128 x i64 + v256i64 = 77, // 256 x i64 + + v1i128 = 78, // 1 x i128 FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i1, LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i128, - v1f16 = 75, // 1 x f16 - v2f16 = 76, // 2 x f16 - v3f16 = 77, // 3 x f16 - v4f16 = 78, // 4 x f16 - v8f16 = 79, // 8 x f16 - v16f16 = 80, // 16 x f16 - v32f16 = 81, // 32 x f16 - v64f16 = 82, // 64 x f16 - v128f16 = 83, // 128 x f16 - v256f16 = 84, // 256 x f16 - v512f16 = 85, // 256 x f16 - - v2bf16 = 86, // 2 x bf16 - v3bf16 = 87, // 3 x bf16 - v4bf16 = 88, // 4 x bf16 - v8bf16 = 89, // 8 x bf16 - v16bf16 = 90, // 16 x bf16 - v32bf16 = 91, // 32 x bf16 - v64bf16 = 92, // 64 x bf16 - v128bf16 = 93, // 128 x bf16 - - v1f32 = 94, // 1 x f32 - v2f32 = 95, // 2 x f32 - v3f32 = 96, // 3 x f32 - v4f32 = 97, // 4 x f32 - v5f32 = 98, // 5 x f32 - v6f32 = 99, // 6 x f32 - v7f32 = 100, // 7 x f32 - v8f32 = 101, // 8 x f32 - v16f32 = 102, // 16 x f32 - v32f32 = 103, // 32 x f32 - v64f32 = 104, // 64 x f32 - v128f32 = 105, // 128 x f32 - v256f32 = 106, // 256 x f32 - v512f32 = 107, // 512 x f32 - v1024f32 = 108, // 1024 x f32 - v2048f32 = 109, // 2048 x f32 - - v1f64 = 110, // 1 x f64 - v2f64 = 111, // 2 x f64 - v3f64 = 112, // 3 x f64 - v4f64 = 113, // 4 x f64 - v8f64 = 114, // 8 x f64 - v16f64 = 115, // 16 x f64 - v32f64 = 116, // 32 x f64 - v64f64 = 117, // 64 x f64 - v128f64 = 118, // 128 x f64 - v256f64 = 119, // 256 x f64 + v1f16 = 79, // 1 x f16 + v2f16 = 80, // 2 x f16 + v3f16 = 81, // 3 x f16 + v4f16 = 82, // 4 x f16 + v8f16 = 83, // 8 x f16 + v16f16 = 84, // 16 x f16 + v32f16 = 85, // 32 x f16 + v64f16 = 86, // 64 x f16 + v128f16 = 87, // 128 x f16 + v256f16 = 88, // 256 x f16 + v512f16 = 89, // 256 x f16 + + v2bf16 = 90, // 2 x bf16 + v3bf16 = 91, // 3 x bf16 + v4bf16 = 92, // 4 x bf16 + v8bf16 = 93, // 8 x bf16 + v16bf16 = 94, // 16 x bf16 + v32bf16 = 95, // 32 x bf16 + v64bf16 = 96, // 64 x bf16 + v128bf16 = 97, // 128 x bf16 + + v1f32 = 98, // 1 x f32 + v2f32 = 99, // 2 x f32 + v3f32 = 100, // 3 x f32 + v4f32 = 101, // 4 x f32 + v5f32 = 102, // 5 x f32 + v6f32 = 103, // 6 x f32 + v7f32 = 104, // 7 x f32 + v8f32 = 105, // 8 x f32 + v16f32 = 106, // 16 x f32 + v32f32 = 107, // 32 x f32 + v64f32 = 108, // 64 x f32 + v128f32 = 109, // 128 x f32 + v256f32 = 110, // 256 x f32 + v512f32 = 111, // 512 x f32 + v1024f32 = 112, // 1024 x f32 + v2048f32 = 113, // 2048 x f32 + + v1f64 = 114, // 1 x f64 + v2f64 = 115, // 2 x f64 + v3f64 = 116, // 3 x f64 + v4f64 = 117, // 4 x f64 + v8f64 = 118, // 8 x f64 + v16f64 = 119, // 16 x f64 + v32f64 = 120, // 32 x f64 + v64f64 = 121, // 64 x f64 + v128f64 = 122, // 128 x f64 + v256f64 = 123, // 256 x f64 FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v1f16, LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v256f64, @@ -185,68 +191,70 @@ namespace llvm { FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1, LAST_FIXEDLEN_VECTOR_VALUETYPE = v256f64, - nxv1i1 = 120, // n x 1 x i1 - nxv2i1 = 121, // n x 2 x i1 - nxv4i1 = 122, // n x 4 x i1 - nxv8i1 = 123, // n x 8 x i1 - nxv16i1 = 124, // n x 16 x i1 - nxv32i1 = 125, // n x 32 x i1 - nxv64i1 = 126, // n x 64 x i1 - - nxv1i8 = 127, // n x 1 x i8 - nxv2i8 = 128, // n x 2 x i8 - nxv4i8 = 129, // n x 4 x i8 - nxv8i8 = 130, // n x 8 x i8 - nxv16i8 = 131, // n x 16 x i8 - nxv32i8 = 132, // n x 32 x i8 - nxv64i8 = 133, // n x 64 x i8 - - nxv1i16 = 134, // n x 1 x i16 - nxv2i16 = 135, // n x 2 x i16 - nxv4i16 = 136, // n x 4 x i16 - nxv8i16 = 137, // n x 8 x i16 - nxv16i16 = 138, // n x 16 x i16 - nxv32i16 = 139, // n x 32 x i16 - - nxv1i32 = 140, // n x 1 x i32 - nxv2i32 = 141, // n x 2 x i32 - nxv4i32 = 142, // n x 4 x i32 - nxv8i32 = 143, // n x 8 x i32 - nxv16i32 = 144, // n x 16 x i32 - nxv32i32 = 145, // n x 32 x i32 - - nxv1i64 = 146, // n x 1 x i64 - nxv2i64 = 147, // n x 2 x i64 - nxv4i64 = 148, // n x 4 x i64 - nxv8i64 = 149, // n x 8 x i64 - nxv16i64 = 150, // n x 16 x i64 - nxv32i64 = 151, // n x 32 x i64 + nxv1i1 = 124, // n x 1 x i1 + nxv2i1 = 125, // n x 2 x i1 + nxv4i1 = 126, // n x 4 x i1 + nxv8i1 = 127, // n x 8 x i1 + nxv16i1 = 128, // n x 16 x i1 + nxv32i1 = 129, // n x 32 x i1 + nxv64i1 = 130, // n x 64 x i1 + + nxv1i8 = 131, // n x 1 x i8 + nxv2i8 = 132, // n x 2 x i8 + nxv4i8 = 133, // n x 4 x i8 + nxv8i8 = 134, // n x 8 x i8 + nxv16i8 = 135, // n x 16 x i8 + nxv32i8 = 136, // n x 32 x i8 + nxv64i8 = 137, // n x 64 x i8 + + nxv1i16 = 138, // n x 1 x i16 + nxv2i16 = 139, // n x 2 x i16 + nxv4i16 = 140, // n x 4 x i16 + nxv8i16 = 141, // n x 8 x i16 + nxv16i16 = 142, // n x 16 x i16 + nxv32i16 = 143, // n x 32 x i16 + + nxv1i32 = 144, // n x 1 x i32 + nxv2i32 = 145, // n x 2 x i32 + nxv4i32 = 146, // n x 4 x i32 + nxv8i32 = 147, // n x 8 x i32 + nxv16i32 = 148, // n x 16 x i32 + nxv32i32 = 149, // n x 32 x i32 + + nxv1i64 = 150, // n x 1 x i64 + nxv2i64 = 151, // n x 2 x i64 + nxv4i64 = 152, // n x 4 x i64 + nxv8i64 = 153, // n x 8 x i64 + nxv16i64 = 154, // n x 16 x i64 + nxv32i64 = 155, // n x 32 x i64 FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1, LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64, - nxv1f16 = 152, // n x 1 x f16 - nxv2f16 = 153, // n x 2 x f16 - nxv4f16 = 154, // n x 4 x f16 - nxv8f16 = 155, // n x 8 x f16 - nxv16f16 = 156, // n x 16 x f16 - nxv32f16 = 157, // n x 32 x f16 - - nxv1bf16 = 158, // n x 1 x bf16 - nxv2bf16 = 159, // n x 2 x bf16 - nxv4bf16 = 160, // n x 4 x bf16 - nxv8bf16 = 161, // n x 8 x bf16 - - nxv1f32 = 162, // n x 1 x f32 - nxv2f32 = 163, // n x 2 x f32 - nxv4f32 = 164, // n x 4 x f32 - nxv8f32 = 165, // n x 8 x f32 - nxv16f32 = 166, // n x 16 x f32 - - nxv1f64 = 167, // n x 1 x f64 - nxv2f64 = 168, // n x 2 x f64 - nxv4f64 = 169, // n x 4 x f64 - nxv8f64 = 170, // n x 8 x f64 + nxv1f16 = 156, // n x 1 x f16 + nxv2f16 = 157, // n x 2 x f16 + nxv4f16 = 158, // n x 4 x f16 + nxv8f16 = 159, // n x 8 x f16 + nxv16f16 = 160, // n x 16 x f16 + nxv32f16 = 161, // n x 32 x f16 + + nxv1bf16 = 162, // n x 1 x bf16 + nxv2bf16 = 163, // n x 2 x bf16 + nxv4bf16 = 164, // n x 4 x bf16 + nxv8bf16 = 165, // n x 8 x bf16 + nxv16bf16 = 166, // n x 16 x bf16 + nxv32bf16 = 167, // n x 32 x bf16 + + nxv1f32 = 168, // n x 1 x f32 + nxv2f32 = 169, // n x 2 x f32 + nxv4f32 = 170, // n x 4 x f32 + nxv8f32 = 171, // n x 8 x f32 + nxv16f32 = 172, // n x 16 x f32 + + nxv1f64 = 173, // n x 1 x f64 + nxv2f64 = 174, // n x 2 x f64 + nxv4f64 = 175, // n x 4 x f64 + nxv8f64 = 176, // n x 8 x f64 FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv1f16, LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64, @@ -257,20 +265,20 @@ namespace llvm { FIRST_VECTOR_VALUETYPE = v1i1, LAST_VECTOR_VALUETYPE = nxv8f64, - x86mmx = 171, // This is an X86 MMX value + x86mmx = 177, // This is an X86 MMX value - Glue = 172, // This glues nodes together during pre-RA sched + Glue = 178, // This glues nodes together during pre-RA sched - isVoid = 173, // This has no value + isVoid = 179, // This has no value - Untyped = 174, // This value takes a register, but has + Untyped = 180, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. - funcref = 175, // WebAssembly's funcref type - externref = 176, // WebAssembly's externref type - x86amx = 177, // This is an X86 AMX value - i64x8 = 178, // 8 Consecutive GPRs (AArch64) + funcref = 181, // WebAssembly's funcref type + externref = 182, // WebAssembly's externref type + x86amx = 183, // This is an X86 AMX value + i64x8 = 184, // 8 Consecutive GPRs (AArch64) FIRST_VALUETYPE = 1, // This is always the beginning of the list. LAST_VALUETYPE = i64x8, // This always remains at the end of the list. @@ -415,10 +423,11 @@ namespace llvm { /// Return true if this is a 256-bit vector type. bool is256BitVector() const { return (SimpleTy == MVT::v16f16 || SimpleTy == MVT::v16bf16 || - SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 || - SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 || - SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64 || - SimpleTy == MVT::v256i1); + SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 || + SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 || + SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64 || + SimpleTy == MVT::v256i1 || SimpleTy == MVT::v128i2 || + SimpleTy == MVT::v64i4); } /// Return true if this is a 512-bit vector type. @@ -517,6 +526,7 @@ namespace llvm { } MVT getVectorElementType() const { + // clang-format off switch (SimpleTy) { default: llvm_unreachable("Not a vector MVT!"); @@ -538,6 +548,8 @@ namespace llvm { case nxv16i1: case nxv32i1: case nxv64i1: return i1; + case v128i2: return i2; + case v64i4: return i4; case v1i8: case v2i8: case v4i8: @@ -640,7 +652,9 @@ namespace llvm { case nxv1bf16: case nxv2bf16: case nxv4bf16: - case nxv8bf16: return bf16; + case nxv8bf16: + case nxv16bf16: + case nxv32bf16: return bf16; case v1f32: case v2f32: case v3f32: @@ -677,6 +691,7 @@ namespace llvm { case nxv4f64: case nxv8f64: return f64; } + // clang-format on } /// Given a vector type, return the minimum number of elements it contains. @@ -705,6 +720,7 @@ namespace llvm { case v256f32: case v256f64: return 256; case v128i1: + case v128i2: case v128i8: case v128i16: case v128i32: @@ -714,6 +730,7 @@ namespace llvm { case v128f32: case v128f64: return 128; case v64i1: + case v64i4: case v64i8: case v64i16: case v64i32: @@ -738,7 +755,8 @@ namespace llvm { case nxv32i16: case nxv32i32: case nxv32i64: - case nxv32f16: return 32; + case nxv32f16: + case nxv32bf16: return 32; case v16i1: case v16i8: case v16i16: @@ -754,6 +772,7 @@ namespace llvm { case nxv16i32: case nxv16i64: case nxv16f16: + case nxv16bf16: case nxv16f32: return 16; case v8i1: case v8i8: @@ -883,8 +902,10 @@ namespace llvm { case i1: case v1i1: return TypeSize::Fixed(1); case nxv1i1: return TypeSize::Scalable(1); + case i2: case v2i1: return TypeSize::Fixed(2); case nxv2i1: return TypeSize::Scalable(2); + case i4: case v4i1: return TypeSize::Fixed(4); case nxv4i1: return TypeSize::Scalable(4); case i8 : @@ -977,6 +998,8 @@ namespace llvm { case v7i32: case v7f32: return TypeSize::Fixed(224); case v256i1: + case v128i2: + case v64i4: case v32i8: case v16i16: case v8i32: @@ -990,6 +1013,7 @@ namespace llvm { case nxv8i32: case nxv4i64: case nxv16f16: + case nxv16bf16: case nxv8f32: case nxv4f64: return TypeSize::Scalable(256); case i64x8: @@ -1007,6 +1031,7 @@ namespace llvm { case nxv16i32: case nxv8i64: case nxv32f16: + case nxv32bf16: case nxv16f32: case nxv8f64: return TypeSize::Scalable(512); case v1024i1: @@ -1078,6 +1103,12 @@ namespace llvm { return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()}; } + // Return the number of bytes overwritten by a store of this value type or + // this value type's element type in the case of a vector. + uint64_t getScalarStoreSize() const { + return getScalarType().getStoreSize().getFixedSize(); + } + /// Return the number of bits overwritten by a store of the specified value /// type. /// @@ -1165,6 +1196,10 @@ namespace llvm { return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); case 1: return MVT::i1; + case 2: + return MVT::i2; + case 4: + return MVT::i4; case 8: return MVT::i8; case 16: @@ -1179,6 +1214,7 @@ namespace llvm { } static MVT getVectorVT(MVT VT, unsigned NumElements) { + // clang-format off switch (VT.SimpleTy) { default: break; @@ -1195,6 +1231,12 @@ namespace llvm { if (NumElements == 512) return MVT::v512i1; if (NumElements == 1024) return MVT::v1024i1; break; + case MVT::i2: + if (NumElements == 128) return MVT::v128i2; + break; + case MVT::i4: + if (NumElements == 64) return MVT::v64i4; + break; case MVT::i8: if (NumElements == 1) return MVT::v1i8; if (NumElements == 2) return MVT::v2i8; @@ -1309,6 +1351,7 @@ namespace llvm { break; } return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); + // clang-format on } static MVT getScalableVectorVT(MVT VT, unsigned NumElements) { @@ -1370,6 +1413,8 @@ namespace llvm { if (NumElements == 2) return MVT::nxv2bf16; if (NumElements == 4) return MVT::nxv4bf16; if (NumElements == 8) return MVT::nxv8bf16; + if (NumElements == 16) return MVT::nxv16bf16; + if (NumElements == 32) return MVT::nxv32bf16; break; case MVT::f32: if (NumElements == 1) return MVT::nxv1f32; diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h index 753b1998c40c..8079aa436933 100644 --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -571,6 +571,33 @@ inline unsigned countPopulation(T Value) { return detail::PopulationCounter<T, sizeof(T)>::count(Value); } +/// Return true if the argument contains a non-empty sequence of ones with the +/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true. +/// If true, \p MaskIdx will specify the index of the lowest set bit and \p +/// MaskLen is updated to specify the length of the mask, else neither are +/// updated. +inline bool isShiftedMask_32(uint32_t Value, unsigned &MaskIdx, + unsigned &MaskLen) { + if (!isShiftedMask_32(Value)) + return false; + MaskIdx = countTrailingZeros(Value); + MaskLen = countPopulation(Value); + return true; +} + +/// Return true if the argument contains a non-empty sequence of ones with the +/// remainder zero (64 bit version.) If true, \p MaskIdx will specify the index +/// of the lowest set bit and \p MaskLen is updated to specify the length of the +/// mask, else neither are updated. +inline bool isShiftedMask_64(uint64_t Value, unsigned &MaskIdx, + unsigned &MaskLen) { + if (!isShiftedMask_64(Value)) + return false; + MaskIdx = countTrailingZeros(Value); + MaskLen = countPopulation(Value); + return true; +} + /// Compile time Log2. /// Valid only for positive powers of two. template <size_t kValue> constexpr inline size_t CTLog2() { @@ -680,7 +707,7 @@ constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) { /// Returns the next power of two (in 64-bits) that is strictly greater than A. /// Returns zero on overflow. -inline uint64_t NextPowerOf2(uint64_t A) { +constexpr inline uint64_t NextPowerOf2(uint64_t A) { A |= (A >> 1); A |= (A >> 2); A |= (A >> 4); @@ -708,27 +735,34 @@ inline uint64_t PowerOf2Ceil(uint64_t A) { /// Returns the next integer (mod 2**64) that is greater than or equal to /// \p Value and is a multiple of \p Align. \p Align must be non-zero. /// -/// If non-zero \p Skew is specified, the return value will be a minimal -/// integer that is greater than or equal to \p Value and equal to -/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than -/// \p Align, its value is adjusted to '\p Skew mod \p Align'. -/// /// Examples: /// \code /// alignTo(5, 8) = 8 /// alignTo(17, 8) = 24 /// alignTo(~0LL, 8) = 0 /// alignTo(321, 255) = 510 +/// \endcode +inline uint64_t alignTo(uint64_t Value, uint64_t Align) { + assert(Align != 0u && "Align can't be 0."); + return (Value + Align - 1) / Align * Align; +} + +/// If non-zero \p Skew is specified, the return value will be a minimal integer +/// that is greater than or equal to \p Size and equal to \p A * N + \p Skew for +/// some integer N. If \p Skew is larger than \p A, its value is adjusted to '\p +/// Skew mod \p A'. \p Align must be non-zero. /// +/// Examples: +/// \code /// alignTo(5, 8, 7) = 7 /// alignTo(17, 8, 1) = 17 /// alignTo(~0LL, 8, 3) = 3 /// alignTo(321, 255, 42) = 552 /// \endcode -inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { +inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew) { assert(Align != 0u && "Align can't be 0."); Skew %= Align; - return (Value + Align - 1 - Skew) / Align * Align + Skew; + return alignTo(Value - Skew, Align) + Skew; } /// Returns the next integer (mod 2**64) that is greater than or equal to @@ -879,7 +913,7 @@ extern const float huge_valf; /// Add two signed integers, computing the two's complement truncated result, -/// returning true if overflow occured. +/// returning true if overflow occurred. template <typename T> std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) { #if __has_builtin(__builtin_add_overflow) diff --git a/llvm/include/llvm/Support/Parallel.h b/llvm/include/llvm/Support/Parallel.h index 04caf5eac961..ff113f9b44c4 100644 --- a/llvm/include/llvm/Support/Parallel.h +++ b/llvm/include/llvm/Support/Parallel.h @@ -193,11 +193,11 @@ void parallelSort(RandomAccessIterator Start, RandomAccessIterator End, llvm::sort(Start, End, Comp); } -void parallelForEachN(size_t Begin, size_t End, function_ref<void(size_t)> Fn); +void parallelFor(size_t Begin, size_t End, function_ref<void(size_t)> Fn); template <class IterTy, class FuncTy> void parallelForEach(IterTy Begin, IterTy End, FuncTy Fn) { - parallelForEachN(0, End - Begin, [&](size_t I) { Fn(Begin[I]); }); + parallelFor(0, End - Begin, [&](size_t I) { Fn(Begin[I]); }); } template <class IterTy, class ResultTy, class ReduceFuncTy, diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h index da5095714f48..ce69f32b6cc8 100644 --- a/llvm/include/llvm/Support/Path.h +++ b/llvm/include/llvm/Support/Path.h @@ -19,7 +19,6 @@ #include "llvm/ADT/iterator.h" #include "llvm/Support/DataTypes.h" #include <iterator> -#include <system_error> namespace llvm { namespace sys { diff --git a/llvm/include/llvm/Support/PluginLoader.h b/llvm/include/llvm/Support/PluginLoader.h index 95c087f03d9b..bdd36366d1cf 100644 --- a/llvm/include/llvm/Support/PluginLoader.h +++ b/llvm/include/llvm/Support/PluginLoader.h @@ -31,9 +31,9 @@ namespace llvm { #ifndef DONT_GET_PLUGIN_LOADER_OPTION // This causes operator= above to be invoked for every -load option. - static cl::opt<PluginLoader, false, cl::parser<std::string> > - LoadOpt("load", cl::ZeroOrMore, cl::value_desc("pluginfilename"), - cl::desc("Load the specified plugin")); + static cl::opt<PluginLoader, false, cl::parser<std::string>> + LoadOpt("load", cl::value_desc("pluginfilename"), + cl::desc("Load the specified plugin")); #endif } diff --git a/llvm/include/llvm/Support/Printable.h b/llvm/include/llvm/Support/Printable.h index 6403c32aad67..8e76f01f6ba2 100644 --- a/llvm/include/llvm/Support/Printable.h +++ b/llvm/include/llvm/Support/Printable.h @@ -24,12 +24,12 @@ class raw_ostream; /// This class is useful to construct print helpers for raw_ostream. /// /// Example: -/// Printable PrintRegister(unsigned Register) { +/// Printable printRegister(unsigned Register) { /// return Printable([Register](raw_ostream &OS) { /// OS << getRegisterName(Register); -/// } +/// }); /// } -/// ... OS << PrintRegister(Register); ... +/// ... OS << printRegister(Register); ... /// /// Implementation note: Ideally this would just be a typedef, but doing so /// leads to operator << being ambiguous as function has matching constructors @@ -47,6 +47,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const Printable &P) { return OS; } -} +} // namespace llvm #endif diff --git a/llvm/include/llvm/Support/Process.h b/llvm/include/llvm/Support/Process.h index ee03efeed9b2..9f56bd9b6e61 100644 --- a/llvm/include/llvm/Support/Process.h +++ b/llvm/include/llvm/Support/Process.h @@ -25,7 +25,6 @@ #define LLVM_SUPPORT_PROCESS_H #include "llvm/ADT/Optional.h" -#include "llvm/Support/AllocatorBase.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/Error.h" diff --git a/llvm/include/llvm/Support/Program.h b/llvm/include/llvm/Support/Program.h index f91fca1c4464..4cb55c42c377 100644 --- a/llvm/include/llvm/Support/Program.h +++ b/llvm/include/llvm/Support/Program.h @@ -14,7 +14,6 @@ #define LLVM_SUPPORT_PROGRAM_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/BitVector.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/Config/llvm-config.h" @@ -24,6 +23,7 @@ #include <system_error> namespace llvm { +class BitVector; namespace sys { /// This is the OS-specific separator for PATH like environment variables: diff --git a/llvm/include/llvm/Support/RISCVISAInfo.h b/llvm/include/llvm/Support/RISCVISAInfo.h index 7fa0e6ee3acf..eac6cc0925fb 100644 --- a/llvm/include/llvm/Support/RISCVISAInfo.h +++ b/llvm/include/llvm/Support/RISCVISAInfo.h @@ -66,6 +66,7 @@ public: bool hasExtension(StringRef Ext) const; std::string toString() const; std::vector<std::string> toFeatureVector() const; + StringRef computeDefaultABI() const; static bool isSupportedExtensionFeature(StringRef Ext); static bool isSupportedExtension(StringRef Ext); @@ -89,6 +90,7 @@ private: Error checkDependency(); void updateImplication(); + void updateCombination(); void updateFLen(); void updateMinVLen(); void updateMaxELen(); diff --git a/llvm/include/llvm/Support/RWMutex.h b/llvm/include/llvm/Support/RWMutex.h index 33a5d3efffee..3dd962586c36 100644 --- a/llvm/include/llvm/Support/RWMutex.h +++ b/llvm/include/llvm/Support/RWMutex.h @@ -93,8 +93,8 @@ private: /// running in multithreaded mode. template <bool mt_only> class SmartRWMutex { // shared_mutex (C++17) is more efficient than shared_timed_mutex (C++14) - // on Windows and always available on MSVC. -#if defined(_MSC_VER) || __cplusplus > 201402L + // on Windows and always available on MSVC except with libc++. +#if (defined(_MSC_VER) && !defined(_LIBCPP_VERSION)) || __cplusplus > 201402L std::shared_mutex impl; #else #if !defined(LLVM_USE_RW_MUTEX_IMPL) diff --git a/llvm/include/llvm/Support/SHA1.h b/llvm/include/llvm/Support/SHA1.h index efd8513cc201..ae6d62aed723 100644 --- a/llvm/include/llvm/Support/SHA1.h +++ b/llvm/include/llvm/Support/SHA1.h @@ -36,17 +36,17 @@ public: /// Digest more data. void update(StringRef Str); - /// Return a reference to the current raw 160-bits SHA1 for the digested data + /// Return the current raw 160-bits SHA1 for the digested data /// since the last call to init(). This call will add data to the internal /// state and as such is not suited for getting an intermediate result /// (see result()). - StringRef final(); + std::array<uint8_t, 20> final(); - /// Return a reference to the current raw 160-bits SHA1 for the digested data + /// Return the current raw 160-bits SHA1 for the digested data /// since the last call to init(). This is suitable for getting the SHA1 at /// any time without invalidating the internal state so that more calls can be /// made into update. - StringRef result(); + std::array<uint8_t, 20> result(); /// Returns a raw 160-bit SHA1 hash for the given data. static std::array<uint8_t, 20> hash(ArrayRef<uint8_t> Data); @@ -68,14 +68,13 @@ private: uint8_t BufferOffset; } InternalState; - // Internal copy of the hash, populated and accessed on calls to result() - uint32_t HashResult[HASH_LENGTH / 4]; - // Helper void writebyte(uint8_t data); void hashBlock(); void addUncounted(uint8_t data); void pad(); + + void final(std::array<uint32_t, HASH_LENGTH / 4> &HashResult); }; } // end llvm namespace diff --git a/llvm/include/llvm/Support/SHA256.h b/llvm/include/llvm/Support/SHA256.h index 9e295b0b9fae..68b32c7b4834 100644 --- a/llvm/include/llvm/Support/SHA256.h +++ b/llvm/include/llvm/Support/SHA256.h @@ -43,17 +43,17 @@ public: /// Digest more data. void update(StringRef Str); - /// Return a reference to the current raw 256-bits SHA256 for the digested + /// Return the current raw 256-bits SHA256 for the digested /// data since the last call to init(). This call will add data to the /// internal state and as such is not suited for getting an intermediate /// result (see result()). - StringRef final(); + std::array<uint8_t, 32> final(); - /// Return a reference to the current raw 256-bits SHA256 for the digested + /// Return the current raw 256-bits SHA256 for the digested /// data since the last call to init(). This is suitable for getting the /// SHA256 at any time without invalidating the internal state so that more /// calls can be made into update. - StringRef result(); + std::array<uint8_t, 32> result(); /// Returns a raw 256-bit SHA256 hash for the given data. static std::array<uint8_t, 32> hash(ArrayRef<uint8_t> Data); @@ -75,14 +75,13 @@ private: uint8_t BufferOffset; } InternalState; - // Internal copy of the hash, populated and accessed on calls to result() - uint32_t HashResult[HASH_LENGTH / 4]; - // Helper void writebyte(uint8_t data); void hashBlock(); void addUncounted(uint8_t data); void pad(); + + void final(std::array<uint32_t, HASH_LENGTH / 4> &HashResult); }; } // namespace llvm diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index 6b5daf710c9f..c9eabfb3788c 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -81,7 +81,6 @@ struct FlagEntry { }; raw_ostream &operator<<(raw_ostream &OS, const HexNumber &Value); -std::string to_hexString(uint64_t Value, bool UpperCase = true); template <class T> std::string to_string(const T &Value) { std::string number; @@ -95,7 +94,7 @@ std::string enumToString(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues) { for (const EnumEntry<TEnum> &EnumItem : EnumValues) if (EnumItem.Value == Value) return std::string(EnumItem.AltName); - return to_hexString(Value, false); + return utohexstr(Value, true); } class ScopedPrinter { @@ -107,7 +106,7 @@ public: ScopedPrinter(raw_ostream &OS, ScopedPrinterKind Kind = ScopedPrinterKind::Base) - : OS(OS), IndentLevel(0), Kind(Kind) {} + : OS(OS), Kind(Kind) {} ScopedPrinterKind getKind() const { return Kind; } @@ -498,7 +497,7 @@ private: } raw_ostream &OS; - int IndentLevel; + int IndentLevel = 0; StringRef Prefix; ScopedPrinterKind Kind; }; diff --git a/llvm/include/llvm/Support/Signals.h b/llvm/include/llvm/Support/Signals.h index 44f5a750ff5c..937e0572d4a7 100644 --- a/llvm/include/llvm/Support/Signals.h +++ b/llvm/include/llvm/Support/Signals.h @@ -14,6 +14,7 @@ #ifndef LLVM_SUPPORT_SIGNALS_H #define LLVM_SUPPORT_SIGNALS_H +#include <cstdint> #include <string> namespace llvm { diff --git a/llvm/include/llvm/Support/Signposts.h b/llvm/include/llvm/Support/Signposts.h index dabbba6f89d1..37089bd1c17d 100644 --- a/llvm/include/llvm/Support/Signposts.h +++ b/llvm/include/llvm/Support/Signposts.h @@ -16,11 +16,11 @@ #ifndef LLVM_SUPPORT_SIGNPOSTS_H #define LLVM_SUPPORT_SIGNPOSTS_H -#include "llvm/ADT/StringRef.h" #include <memory> namespace llvm { class SignpostEmitterImpl; +class StringRef; /// Manages the emission of signposts into the recording method supported by /// the OS. diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h index 28716b42f4ab..eced4574c82e 100644 --- a/llvm/include/llvm/Support/SourceMgr.h +++ b/llvm/include/llvm/Support/SourceMgr.h @@ -100,6 +100,9 @@ public: SourceMgr &operator=(SourceMgr &&) = default; ~SourceMgr() = default; + /// Return the include directories of this source manager. + ArrayRef<std::string> getIncludeDirs() const { return IncludeDirectories; } + void setIncludeDirs(const std::vector<std::string> &Dirs) { IncludeDirectories = Dirs; } @@ -147,6 +150,22 @@ public: return Buffers.size(); } + /// Takes the source buffers from the given source manager and append them to + /// the current manager. `MainBufferIncludeLoc` is an optional include + /// location to attach to the main buffer of `SrcMgr` after it gets moved to + /// the current manager. + void takeSourceBuffersFrom(SourceMgr &SrcMgr, + SMLoc MainBufferIncludeLoc = SMLoc()) { + if (SrcMgr.Buffers.empty()) + return; + + size_t OldNumBuffers = getNumBuffers(); + std::move(SrcMgr.Buffers.begin(), SrcMgr.Buffers.end(), + std::back_inserter(Buffers)); + SrcMgr.Buffers.clear(); + Buffers[OldNumBuffers].IncludeLoc = MainBufferIncludeLoc; + } + /// Search for a file with the specified name in the current directory or in /// one of the IncludeDirs. /// @@ -156,6 +175,17 @@ public: unsigned AddIncludeFile(const std::string &Filename, SMLoc IncludeLoc, std::string &IncludedFile); + /// Search for a file with the specified name in the current directory or in + /// one of the IncludeDirs, and try to open it **without** adding to the + /// SourceMgr. If the opened file is intended to be added to the source + /// manager, prefer `AddIncludeFile` instead. + /// + /// If no file is found, this returns an Error, otherwise it returns the + /// buffer of the stacked file. The full path to the included file can be + /// found in \p IncludedFile. + ErrorOr<std::unique_ptr<MemoryBuffer>> + OpenIncludeFile(const std::string &Filename, std::string &IncludedFile); + /// Return the ID of the buffer containing the specified location. /// /// 0 is returned if the buffer is not found. diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 428cbb44705d..8df7ced0029d 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -322,6 +322,9 @@ HANDLE_TARGET_OPCODE(G_BITCAST) /// Generic freeze. HANDLE_TARGET_OPCODE(G_FREEZE) +// INTRINSIC fptrunc_round intrinsic. +HANDLE_TARGET_OPCODE(G_INTRINSIC_FPTRUNC_ROUND) + /// INTRINSIC trunc intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_TRUNC) @@ -617,6 +620,9 @@ HANDLE_TARGET_OPCODE(G_FABS) /// f64) is allowed. HANDLE_TARGET_OPCODE(G_FCOPYSIGN) +/// Generic test for floating-point class. +HANDLE_TARGET_OPCODE(G_IS_FPCLASS) + /// Generic FP canonicalize value. HANDLE_TARGET_OPCODE(G_FCANONICALIZE) diff --git a/llvm/include/llvm/Support/TargetParser.h b/llvm/include/llvm/Support/TargetParser.h index 02a8d72483db..c3a6cceaee6b 100644 --- a/llvm/include/llvm/Support/TargetParser.h +++ b/llvm/include/llvm/Support/TargetParser.h @@ -14,14 +14,14 @@ #ifndef LLVM_SUPPORT_TARGETPARSER_H #define LLVM_SUPPORT_TARGETPARSER_H +#include "llvm/ADT/StringRef.h" +#include <cstdint> // FIXME: vector is used because that's what clang uses for subtarget feature // lists, but SmallVector would probably be better -#include "llvm/Support/RISCVISAInfo.h" #include <vector> namespace llvm { -class StringRef; template <typename T> class SmallVectorImpl; class Triple; @@ -86,6 +86,7 @@ enum GPUKind : uint32_t { GK_GFX909 = 65, GK_GFX90A = 66, GK_GFX90C = 67, + GK_GFX940 = 68, GK_GFX1010 = 71, GK_GFX1011 = 72, @@ -97,9 +98,15 @@ enum GPUKind : uint32_t { GK_GFX1033 = 78, GK_GFX1034 = 79, GK_GFX1035 = 80, + GK_GFX1036 = 81, + + GK_GFX1100 = 90, + GK_GFX1101 = 91, + GK_GFX1102 = 92, + GK_GFX1103 = 93, GK_AMDGCN_FIRST = GK_GFX600, - GK_AMDGCN_LAST = GK_GFX1035, + GK_AMDGCN_LAST = GK_GFX1103, }; /// Instruction set architecture version. @@ -170,7 +177,6 @@ void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64); void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64); bool getCPUFeaturesExceptStdExt(CPUKind Kind, std::vector<StringRef> &Features); StringRef resolveTuneCPUAlias(StringRef TuneCPU, bool IsRV64); -StringRef computeDefaultABIFromArch(const llvm::RISCVISAInfo &ISAInfo); } // namespace RISCV diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h index 868dd2819f83..5e67a312d5c7 100644 --- a/llvm/include/llvm/Support/ThreadPool.h +++ b/llvm/include/llvm/Support/ThreadPool.h @@ -13,26 +13,42 @@ #ifndef LLVM_SUPPORT_THREADPOOL_H #define LLVM_SUPPORT_THREADPOOL_H +#include "llvm/ADT/DenseMap.h" #include "llvm/Config/llvm-config.h" +#include "llvm/Support/RWMutex.h" #include "llvm/Support/Threading.h" #include "llvm/Support/thread.h" #include <future> #include <condition_variable> +#include <deque> #include <functional> #include <memory> #include <mutex> -#include <queue> #include <utility> namespace llvm { +class ThreadPoolTaskGroup; + /// A ThreadPool for asynchronous parallel execution on a defined number of /// threads. /// /// The pool keeps a vector of threads alive, waiting on a condition variable /// for some work to become available. +/// +/// It is possible to reuse one thread pool for different groups of tasks +/// by grouping tasks using ThreadPoolTaskGroup. All tasks are processed using +/// the same queue, but it is possible to wait only for a specific group of +/// tasks to finish. +/// +/// It is also possible for worker threads to submit new tasks and wait for +/// them. Note that this may result in a deadlock in cases such as when a task +/// (directly or indirectly) tries to wait for its own completion, or when all +/// available threads are used up by tasks waiting for a task that has no thread +/// left to run on (this includes waiting on the returned future). It should be +/// generally safe to wait() for a group as long as groups do not form a cycle. class ThreadPool { public: /// Construct a pool using the hardware strategy \p S for mapping hardware @@ -47,23 +63,47 @@ public: /// Asynchronous submission of a task to the pool. The returned future can be /// used to wait for the task to finish and is *non-blocking* on destruction. template <typename Function, typename... Args> - inline auto async(Function &&F, Args &&...ArgList) { + auto async(Function &&F, Args &&...ArgList) { auto Task = std::bind(std::forward<Function>(F), std::forward<Args>(ArgList)...); return async(std::move(Task)); } + /// Overload, task will be in the given task group. + template <typename Function, typename... Args> + auto async(ThreadPoolTaskGroup &Group, Function &&F, Args &&...ArgList) { + auto Task = + std::bind(std::forward<Function>(F), std::forward<Args>(ArgList)...); + return async(Group, std::move(Task)); + } + /// Asynchronous submission of a task to the pool. The returned future can be /// used to wait for the task to finish and is *non-blocking* on destruction. template <typename Func> auto async(Func &&F) -> std::shared_future<decltype(F())> { - return asyncImpl(std::function<decltype(F())()>(std::forward<Func>(F))); + return asyncImpl(std::function<decltype(F())()>(std::forward<Func>(F)), + nullptr); + } + + template <typename Func> + auto async(ThreadPoolTaskGroup &Group, Func &&F) + -> std::shared_future<decltype(F())> { + return asyncImpl(std::function<decltype(F())()>(std::forward<Func>(F)), + &Group); } /// Blocking wait for all the threads to complete and the queue to be empty. /// It is an error to try to add new tasks while blocking on this call. + /// Calling wait() from a task would deadlock waiting for itself. void wait(); + /// Blocking wait for only all the threads in the given group to complete. + /// It is possible to wait even inside a task, but waiting (directly or + /// indirectly) on itself will deadlock. If called from a task running on a + /// worker thread, the call may process pending tasks while waiting in order + /// not to waste the thread. + void wait(ThreadPoolTaskGroup &Group); + // TODO: misleading legacy name warning! // Returns the maximum number of worker threads in the pool, not the current // number of threads! @@ -98,12 +138,15 @@ private: std::move(F)}; } - bool workCompletedUnlocked() { return !ActiveThreads && Tasks.empty(); } + /// Returns true if all tasks in the given group have finished (nullptr means + /// all tasks regardless of their group). QueueLock must be locked. + bool workCompletedUnlocked(ThreadPoolTaskGroup *Group) const; /// Asynchronous submission of a task to the pool. The returned future can be /// used to wait for the task to finish and is *non-blocking* on destruction. template <typename ResTy> - std::shared_future<ResTy> asyncImpl(std::function<ResTy()> Task) { + std::shared_future<ResTy> asyncImpl(std::function<ResTy()> Task, + ThreadPoolTaskGroup *Group) { #if LLVM_ENABLE_THREADS /// Wrap the Task in a std::function<void()> that sets the result of the @@ -117,7 +160,7 @@ private: // Don't allow enqueueing after disabling the pool assert(EnableFlag && "Queuing a thread during ThreadPool destruction"); - Tasks.push(std::move(R.first)); + Tasks.emplace_back(std::make_pair(std::move(R.first), Group)); requestedThreads = ActiveThreads + Tasks.size(); } QueueCondition.notify_one(); @@ -130,7 +173,7 @@ private: auto Future = std::async(std::launch::deferred, std::move(Task)).share(); // Wrap the future so that both ThreadPool::wait() can operate and the // returned future can be sync'ed on. - Tasks.push([Future]() { Future.get(); }); + Tasks.emplace_back(std::make_pair([Future]() { Future.get(); }, Group)); return Future; #endif } @@ -139,25 +182,29 @@ private: // Grow to ensure that we have at least `requested` Threads, but do not go // over MaxThreadCount. void grow(int requested); + + void processTasks(ThreadPoolTaskGroup *WaitingForGroup); #endif /// Threads in flight std::vector<llvm::thread> Threads; /// Lock protecting access to the Threads vector. - mutable std::mutex ThreadsLock; + mutable llvm::sys::RWMutex ThreadsLock; /// Tasks waiting for execution in the pool. - std::queue<std::function<void()>> Tasks; + std::deque<std::pair<std::function<void()>, ThreadPoolTaskGroup *>> Tasks; /// Locking and signaling for accessing the Tasks queue. std::mutex QueueLock; std::condition_variable QueueCondition; - /// Signaling for job completion + /// Signaling for job completion (all tasks or all tasks in a group). std::condition_variable CompletionCondition; /// Keep track of the number of thread actually busy unsigned ActiveThreads = 0; + /// Number of threads active for tasks in the given group (only non-zero). + DenseMap<ThreadPoolTaskGroup *, unsigned> ActiveGroups; #if LLVM_ENABLE_THREADS // avoids warning for unused variable /// Signal for the destruction of the pool, asking thread to exit. @@ -169,6 +216,34 @@ private: /// Maximum number of threads to potentially grow this pool to. const unsigned MaxThreadCount; }; -} + +/// A group of tasks to be run on a thread pool. Thread pool tasks in different +/// groups can run on the same threadpool but can be waited for separately. +/// It is even possible for tasks of one group to submit and wait for tasks +/// of another group, as long as this does not form a loop. +class ThreadPoolTaskGroup { +public: + /// The ThreadPool argument is the thread pool to forward calls to. + ThreadPoolTaskGroup(ThreadPool &Pool) : Pool(Pool) {} + + /// Blocking destructor: will wait for all the tasks in the group to complete + /// by calling ThreadPool::wait(). + ~ThreadPoolTaskGroup() { wait(); } + + /// Calls ThreadPool::async() for this group. + template <typename Function, typename... Args> + inline auto async(Function &&F, Args &&...ArgList) { + return Pool.async(*this, std::forward<Function>(F), + std::forward<Args>(ArgList)...); + } + + /// Calls ThreadPool::wait() for this group. + void wait() { Pool.wait(*this); } + +private: + ThreadPool &Pool; +}; + +} // namespace llvm #endif // LLVM_SUPPORT_THREADPOOL_H diff --git a/llvm/include/llvm/Support/Threading.h b/llvm/include/llvm/Support/Threading.h index 94de950d4470..1e7e5f7b8f50 100644 --- a/llvm/include/llvm/Support/Threading.h +++ b/llvm/include/llvm/Support/Threading.h @@ -15,13 +15,10 @@ #define LLVM_SUPPORT_THREADING_H #include "llvm/ADT/BitVector.h" -#include "llvm/ADT/FunctionExtras.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX #include "llvm/Support/Compiler.h" #include <ciso646> // So we can check the C++ standard lib macros. -#include <functional> #if defined(_MSC_VER) // MSVC's call_once implementation worked since VS 2015, which is the minimum @@ -236,15 +233,20 @@ bool llvm_is_multithreaded(); unsigned get_cpus(); enum class ThreadPriority { + /// Lower the current thread's priority as much as possible. Can be used + /// for long-running tasks that are not time critical; more energy- + /// efficient than Low. Background = 0, - Default = 1, + + /// Lower the current thread's priority such that it does not affect + /// foreground tasks significantly. This is a good default for long- + /// running, latency-insensitive tasks to make sure cpu is not hogged + /// by this task. + Low = 1, + + /// Restore the current thread's priority to default scheduling priority. + Default = 2, }; - /// If priority is Background tries to lower current threads priority such - /// that it does not affect foreground tasks significantly. Can be used for - /// long-running, latency-insensitive tasks to make sure cpu is not hogged by - /// this task. - /// If the priority is default tries to restore current threads priority to - /// default scheduling priority. enum class SetThreadPriorityResult { FAILURE, SUCCESS }; SetThreadPriorityResult set_thread_priority(ThreadPriority Priority); } diff --git a/llvm/include/llvm/Support/TrigramIndex.h b/llvm/include/llvm/Support/TrigramIndex.h index f772deca0301..0bfac498393f 100644 --- a/llvm/include/llvm/Support/TrigramIndex.h +++ b/llvm/include/llvm/Support/TrigramIndex.h @@ -27,12 +27,12 @@ #define LLVM_SUPPORT_TRIGRAMINDEX_H #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" #include <string> #include <unordered_map> #include <vector> namespace llvm { +class StringRef; class TrigramIndex { public: diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 6bddb602e8c1..0b40e970e8c9 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -362,12 +362,31 @@ public: LinearPolySize::get(getKnownMinValue() / RHS, isScalable())); } + LeafTy multiplyCoefficientBy(ScalarTy RHS) const { + return static_cast<LeafTy>( + LinearPolySize::get(getKnownMinValue() * RHS, isScalable())); + } + LeafTy coefficientNextPowerOf2() const { return static_cast<LeafTy>(LinearPolySize::get( static_cast<ScalarTy>(llvm::NextPowerOf2(getKnownMinValue())), isScalable())); } + /// Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) + /// will result in a value whose size matches our own. + bool hasKnownScalarFactor(const LinearPolySize &RHS) const { + return isScalable() == RHS.isScalable() && + getKnownMinValue() % RHS.getKnownMinValue() == 0; + } + + /// Returns a value X where RHS.multiplyCoefficientBy(X) will result in a + /// value whose size matches our own. + ScalarTy getKnownScalarFactor(const LinearPolySize &RHS) const { + assert(hasKnownScalarFactor(RHS) && "Expected RHS to be a known factor!"); + return getKnownMinValue() / RHS.getKnownMinValue(); + } + /// Printing function. void print(raw_ostream &OS) const { if (isScalable()) diff --git a/llvm/include/llvm/Support/Unicode.h b/llvm/include/llvm/Support/Unicode.h index ca17bba2fbb4..729775431e16 100644 --- a/llvm/include/llvm/Support/Unicode.h +++ b/llvm/include/llvm/Support/Unicode.h @@ -14,6 +14,10 @@ #ifndef LLVM_SUPPORT_UNICODE_H #define LLVM_SUPPORT_UNICODE_H +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallString.h" +#include <string> + namespace llvm { class StringRef; @@ -30,19 +34,13 @@ enum ColumnWidthErrors { /// terminal, so we define the semantic that should be suitable for generic case /// of a terminal capable to output Unicode characters. /// -/// All characters from the Unicode code point range are considered printable -/// except for: -/// * C0 and C1 control character ranges; -/// * default ignorable code points as per 5.21 of -/// http://www.unicode.org/versions/Unicode6.2.0/UnicodeStandard-6.2.pdf -/// except for U+00AD SOFT HYPHEN, as it's actually displayed on most -/// terminals; -/// * format characters (category = Cf); -/// * surrogates (category = Cs); -/// * unassigned characters (category = Cn). +/// Printable codepoints are those in the categories L, M, N, P, S and Zs /// \return true if the character is considered printable. bool isPrintable(int UCS); +// Formatting codepoints are codepoints in the Cf category. +bool isFormatting(int UCS); + /// Gets the number of positions the UTF8-encoded \p Text is likely to occupy /// when output on a terminal ("character width"). This depends on the /// implementation of the terminal, and there's no standard definition of @@ -63,6 +61,30 @@ int columnWidthUTF8(StringRef Text); /// rules. int foldCharSimple(int C); +/// Maps the name or the alias of a Unicode character to its associated +/// codepoints. +/// The names and aliases are derived from UnicodeData.txt and NameAliases.txt +/// For compatibility with the semantics of named character escape sequences in +/// C++, this mapping does an exact match sensitive to casing and spacing. +/// \return The codepoint of the corresponding character, if any. +Optional<char32_t> nameToCodepointStrict(StringRef Name); + +struct LooseMatchingResult { + char32_t CodePoint; + SmallString<64> Name; +}; + +Optional<LooseMatchingResult> nameToCodepointLooseMatching(StringRef Name); + +struct MatchForCodepointName { + std::string Name; + uint32_t Distance = 0; + char32_t Value = 0; +}; + +SmallVector<MatchForCodepointName> +nearestMatchesForCodepointName(StringRef Pattern, std::size_t MaxMatchesCount); + } // namespace unicode } // namespace sys } // namespace llvm diff --git a/llvm/include/llvm/Support/VersionTuple.h b/llvm/include/llvm/Support/VersionTuple.h index 1a1072d228f1..2020a5c06f56 100644 --- a/llvm/include/llvm/Support/VersionTuple.h +++ b/llvm/include/llvm/Support/VersionTuple.h @@ -17,11 +17,13 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/Optional.h" -#include "llvm/Support/HashBuilder.h" +#include "llvm/Support/Endian.h" #include <string> #include <tuple> namespace llvm { +template <typename HasherT, support::endianness Endianness> +class HashBuilderImpl; class raw_ostream; class StringRef; @@ -97,6 +99,12 @@ public: return *this; } + /// Return a version tuple that contains a different major version but + /// everything else is the same. + VersionTuple withMajorReplaced(unsigned NewMajor) const { + return VersionTuple(NewMajor, Minor, Subminor, Build); + } + /// Return a version tuple that contains only components that are non-zero. VersionTuple normalize() const { VersionTuple Result = *this; @@ -161,8 +169,8 @@ public: return !(X < Y); } - friend llvm::hash_code hash_value(const VersionTuple &VT) { - return llvm::hash_combine(VT.Major, VT.Minor, VT.Subminor, VT.Build); + friend hash_code hash_value(const VersionTuple &VT) { + return hash_combine(VT.Major, VT.Minor, VT.Subminor, VT.Build); } template <typename HasherT, llvm::support::endianness Endianness> diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index f5dde334b0a7..3c99b0d8efdb 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -22,6 +22,7 @@ #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/ErrorOr.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" @@ -58,6 +59,17 @@ public: // FIXME: remove when files support multiple names bool IsVFSMapped = false; + /// Whether this entity has an external path different from the virtual path, + /// and the external path is exposed by leaking it through the abstraction. + /// For example, a RedirectingFileSystem will set this for paths where + /// UseExternalName is true. + /// + /// FIXME: Currently the external path is exposed by replacing the virtual + /// path in this Status object. Instead, we should leave the path in the + /// Status intact (matching the requested virtual path) - see + /// FileManager::getFileRef for how how we plan to fix this. + bool ExposesExternalVFSPath = false; + Status() = default; Status(const llvm::sys::fs::file_status &Status); Status(const Twine &Name, llvm::sys::fs::UniqueID UID, @@ -306,6 +318,28 @@ public: /// \returns success if \a path has been made absolute, otherwise a /// platform-specific error_code. virtual std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const; + + enum class PrintType { Summary, Contents, RecursiveContents }; + void print(raw_ostream &OS, PrintType Type = PrintType::Contents, + unsigned IndentLevel = 0) const { + printImpl(OS, Type, IndentLevel); + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif + +protected: + virtual void printImpl(raw_ostream &OS, PrintType Type, + unsigned IndentLevel) const { + printIndent(OS, IndentLevel); + OS << "FileSystem\n"; + } + + void printIndent(raw_ostream &OS, unsigned IndentLevel) const { + for (unsigned i = 0; i < IndentLevel; ++i) + OS << " "; + } }; /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by @@ -357,6 +391,8 @@ public: using const_iterator = FileSystemList::const_reverse_iterator; using reverse_iterator = FileSystemList::iterator; using const_reverse_iterator = FileSystemList::const_iterator; + using range = iterator_range<iterator>; + using const_range = iterator_range<const_iterator>; /// Get an iterator pointing to the most recently added file system. iterator overlays_begin() { return FSList.rbegin(); } @@ -373,6 +409,13 @@ public: /// Get an iterator pointing one-past the most recently added file system. reverse_iterator overlays_rend() { return FSList.end(); } const_reverse_iterator overlays_rend() const { return FSList.end(); } + + range overlays_range() { return llvm::reverse(FSList); } + const_range overlays_range() const { return llvm::reverse(FSList); } + +protected: + void printImpl(raw_ostream &OS, PrintType Type, + unsigned IndentLevel) const override; }; /// By default, this delegates all calls to the underlying file system. This @@ -436,6 +479,24 @@ struct NewInMemoryNodeInfo { Status makeStatus() const; }; +class NamedNodeOrError { + ErrorOr<std::pair<llvm::SmallString<128>, const detail::InMemoryNode *>> + Value; + +public: + NamedNodeOrError(llvm::SmallString<128> Name, + const detail::InMemoryNode *Node) + : Value(std::make_pair(Name, Node)) {} + NamedNodeOrError(std::error_code EC) : Value(EC) {} + NamedNodeOrError(llvm::errc EC) : Value(EC) {} + + StringRef getName() const { return (*Value).first; } + explicit operator bool() const { return static_cast<bool>(Value); } + operator std::error_code() const { return Value.getError(); } + std::error_code getError() const { return Value.getError(); } + const detail::InMemoryNode *operator*() const { return (*Value).second; } +}; + } // namespace detail /// An in-memory file system. @@ -454,6 +515,14 @@ class InMemoryFileSystem : public FileSystem { Optional<llvm::sys::fs::file_type> Type, Optional<llvm::sys::fs::perms> Perms, MakeNodeFn MakeNode); + /// Looks up the in-memory node for the path \param P. + /// If \param FollowFinalSymlink is true, the returned node is guaranteed to + /// not be a symlink and its path may differ from \param P. + detail::NamedNodeOrError lookupNode(const Twine &P, bool FollowFinalSymlink, + size_t SymlinkDepth = 0) const; + + class DirIterator; + public: explicit InMemoryFileSystem(bool UseNormalizedPaths = true); ~InMemoryFileSystem() override; @@ -471,18 +540,32 @@ public: Optional<llvm::sys::fs::perms> Perms = None); /// Add a hard link to a file. + /// /// Here hard links are not intended to be fully equivalent to the classical /// filesystem. Both the hard link and the file share the same buffer and /// status (and thus have the same UniqueID). Because of this there is no way /// to distinguish between the link and the file after the link has been /// added. /// - /// The To path must be an existing file or a hardlink. The From file must not - /// have been added before. The To Path must not be a directory. The From Node - /// is added as a hard link which points to the resolved file of To Node. + /// The \param Target path must be an existing file or a hardlink. The + /// \param NewLink file must not have been added before. The \param Target + /// path must not be a directory. The \param NewLink node is added as a hard + /// link which points to the resolved file of \param Target node. /// \return true if the above condition is satisfied and hardlink was /// successfully created, false otherwise. - bool addHardLink(const Twine &From, const Twine &To); + bool addHardLink(const Twine &NewLink, const Twine &Target); + + /// Arbitrary max depth to search through symlinks. We can get into problems + /// if a link links to a link that links back to the link, for example. + static constexpr size_t MaxSymlinkDepth = 16; + + /// Add a symbolic link. Unlike a HardLink, because \param Target doesn't need + /// to refer to a file (or refer to anything, as it happens). Also, an + /// in-memory directory for \param Target isn't automatically created. + bool addSymbolicLink(const Twine &NewLink, const Twine &Target, + time_t ModificationTime, Optional<uint32_t> User = None, + Optional<uint32_t> Group = None, + Optional<llvm::sys::fs::perms> Perms = None); /// Add a buffer to the VFS with a path. The VFS does not own the buffer. /// If present, User, Group, Type and Perms apply to the newly-created file @@ -520,6 +603,10 @@ public: SmallVectorImpl<char> &Output) const override; std::error_code isLocal(const Twine &Path, bool &Result) override; std::error_code setCurrentWorkingDirectory(const Twine &Path) override; + +protected: + void printImpl(raw_ostream &OS, PrintType Type, + unsigned IndentLevel) const override; }; /// Get a globally unique ID for a virtual file or directory. @@ -571,7 +658,10 @@ class RedirectingFileSystemParser; /// 'case-sensitive': <boolean, default=(true for Posix, false for Windows)> /// 'use-external-names': <boolean, default=true> /// 'overlay-relative': <boolean, default=false> -/// 'fallthrough': <boolean, default=true> +/// 'fallthrough': <boolean, default=true, deprecated - use 'redirecting-with' +/// instead> +/// 'redirecting-with': <string, one of 'fallthrough', 'fallback', or +/// 'redirect-only', default='fallthrough'> /// /// Virtual directories that list their contents are represented as /// \verbatim @@ -642,6 +732,20 @@ public: enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File }; enum NameKind { NK_NotSet, NK_External, NK_Virtual }; + /// The type of redirection to perform. + enum class RedirectKind { + /// Lookup the redirected path first (ie. the one specified in + /// 'external-contents') and if that fails "fallthrough" to a lookup of the + /// originally provided path. + Fallthrough, + /// Lookup the provided path first and if that fails, "fallback" to a + /// lookup of the redirected path. + Fallback, + /// Only lookup the redirected path, do not lookup the originally provided + /// path. + RedirectOnly + }; + /// A single file or directory in the VFS. class Entry { EntryKind Kind; @@ -776,17 +880,11 @@ private: friend class RedirectingFSDirIterImpl; friend class RedirectingFileSystemParser; - bool shouldUseExternalFS() const { return IsFallthrough; } - /// Canonicalize path by removing ".", "..", "./", components. This is /// a VFS request, do not bother about symlinks in the path components /// but canonicalize in order to perform the correct entry search. std::error_code makeCanonical(SmallVectorImpl<char> &Path) const; - /// Whether to fall back to the external file system when an operation fails - /// with the given error code on a path associated with the provided Entry. - bool shouldFallBackToExternalFS(std::error_code EC, Entry *E = nullptr) const; - /// Get the File status, or error, from the underlying external file system. /// This returns the status with the originally requested name, while looking /// up the entry using the canonical path. @@ -834,9 +932,9 @@ private: /// names of files. This global value is overridable on a per-file basis. bool UseExternalNames = true; - /// Whether to attempt a file lookup in external file system after it wasn't - /// found in VFS. - bool IsFallthrough = true; + /// Determines the lookups to perform, as well as their order. See + /// \c RedirectKind for details. + RedirectKind Redirection = RedirectKind::Fallthrough; /// @} RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS); @@ -891,15 +989,19 @@ public: StringRef getExternalContentsPrefixDir() const; + /// Sets the redirection kind to \c Fallthrough if true or \c RedirectOnly + /// otherwise. Will removed in the future, use \c setRedirection instead. void setFallthrough(bool Fallthrough); + void setRedirection(RedirectingFileSystem::RedirectKind Kind); + std::vector<llvm::StringRef> getRoots() const; - void dump(raw_ostream &OS) const; - void dumpEntry(raw_ostream &OS, Entry *E, int NumSpaces = 0) const; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const; -#endif + void printEntry(raw_ostream &OS, Entry *E, unsigned IndentLevel = 0) const; + +protected: + void printImpl(raw_ostream &OS, PrintType Type, + unsigned IndentLevel) const override; }; /// Collect all pairs of <virtual path, real path> entries from the diff --git a/llvm/include/llvm/Support/Win64EH.h b/llvm/include/llvm/Support/Win64EH.h index 9359fcb4286a..31345beaa66a 100644 --- a/llvm/include/llvm/Support/Win64EH.h +++ b/llvm/include/llvm/Support/Win64EH.h @@ -24,6 +24,9 @@ namespace Win64EH { /// UnwindOpcodes - Enumeration whose values specify a single operation in /// the prolog of a function. enum UnwindOpcodes { + // The following set of unwind opcodes is for x86_64. They are documented at + // https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64. + // Some generic values from this set are used for other architectures too. UOP_PushNonVol = 0, UOP_AllocLarge, UOP_AllocSmall, @@ -57,7 +60,38 @@ enum UnwindOpcodes { UOP_SaveNext, UOP_TrapFrame, UOP_Context, - UOP_ClearUnwoundToCall + UOP_ClearUnwoundToCall, + // The following set of unwind opcodes is for ARM. They are documented at + // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling + + // Stack allocations use UOP_AllocSmall, UOP_AllocLarge from above, plus + // the following. AllocSmall, AllocLarge and AllocHuge represent a 16 bit + // instruction, while the WideAlloc* opcodes represent a 32 bit instruction. + // Small can represent a stack offset of 0x7f*4 (252) bytes, Medium can + // represent up to 0x3ff*4 (4092) bytes, Large up to 0xffff*4 (262140) bytes, + // and Huge up to 0xffffff*4 (67108860) bytes. + UOP_AllocHuge, + UOP_WideAllocMedium, + UOP_WideAllocLarge, + UOP_WideAllocHuge, + + UOP_WideSaveRegMask, + UOP_SaveSP, + UOP_SaveRegsR4R7LR, + UOP_WideSaveRegsR4R11LR, + UOP_SaveFRegD8D15, + UOP_SaveRegMask, + UOP_SaveLR, + UOP_SaveFRegD0D15, + UOP_SaveFRegD16D31, + // Using UOP_Nop from above + UOP_WideNop, + // Using UOP_End from above + UOP_EndNop, + UOP_WideEndNop, + // A custom unspecified opcode, consisting of one or more bytes. This + // allows producing opcodes in the implementation defined/reserved range. + UOP_Custom, }; /// UnwindCode - This union describes a single operation in a function prolog, diff --git a/llvm/include/llvm/Support/WithColor.h b/llvm/include/llvm/Support/WithColor.h index e772ea667f4f..b249f34da1fa 100644 --- a/llvm/include/llvm/Support/WithColor.h +++ b/llvm/include/llvm/Support/WithColor.h @@ -51,10 +51,9 @@ enum class ColorMode { /// An RAII object that temporarily switches an output stream to a specific /// color. class WithColor { - raw_ostream &OS; - ColorMode Mode; - public: + using AutoDetectFunctionType = bool (*)(const raw_ostream &OS); + /// To be used like this: WithColor(OS, HighlightColor::String) << "text"; /// @param OS The output stream /// @param S Symbolic name for syntax element to color @@ -132,6 +131,19 @@ public: /// Implement default handling for Warning. /// Print "warning: " to stderr. static void defaultWarningHandler(Error Warning); + + /// Retrieve the default color auto detection function. + static AutoDetectFunctionType defaultAutoDetectFunction(); + + /// Change the global auto detection function. + static void + setAutoDetectFunction(AutoDetectFunctionType NewAutoDetectFunction); + +private: + raw_ostream &OS; + ColorMode Mode; + + static AutoDetectFunctionType AutoDetectFunction; }; } // end namespace llvm diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h index aca717a9f6cb..169b8e97986e 100644 --- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -120,8 +120,6 @@ enum attributeBits { ENUM_ENTRY(IC_VEX_XS, 2, "requires VEX and the XS prefix") \ ENUM_ENTRY(IC_VEX_XD, 2, "requires VEX and the XD prefix") \ ENUM_ENTRY(IC_VEX_OPSIZE, 2, "requires VEX and the OpSize prefix") \ - ENUM_ENTRY(IC_64BIT_VEX_OPSIZE, 4, "requires 64-bit mode and VEX") \ - ENUM_ENTRY(IC_64BIT_VEX_OPSIZE_ADSIZE, 5, "requires 64-bit mode, VEX, and AdSize")\ ENUM_ENTRY(IC_VEX_W, 3, "requires VEX and the W prefix") \ ENUM_ENTRY(IC_VEX_W_XS, 4, "requires VEX, W, and XS prefix") \ ENUM_ENTRY(IC_VEX_W_XD, 4, "requires VEX, W, and XD prefix") \ diff --git a/llvm/include/llvm/Support/X86TargetParser.def b/llvm/include/llvm/Support/X86TargetParser.def index 4443d822d3e8..58fa3b3842e7 100644 --- a/llvm/include/llvm/Support/X86TargetParser.def +++ b/llvm/include/llvm/Support/X86TargetParser.def @@ -211,47 +211,47 @@ X86_FEATURE (LVI_LOAD_HARDENING, "lvi-load-hardening") #undef X86_FEATURE #ifndef CPU_SPECIFIC -#define CPU_SPECIFIC(NAME, MANGLING, FEATURES) +#define CPU_SPECIFIC(NAME, TUNE_NAME, MANGLING, FEATURES) #endif #ifndef CPU_SPECIFIC_ALIAS -#define CPU_SPECIFIC_ALIAS(NEW_NAME, NAME) +#define CPU_SPECIFIC_ALIAS(NEW_NAME, TUNE_NAME, NAME) #endif -CPU_SPECIFIC("generic", 'A', "") -CPU_SPECIFIC("pentium", 'B', "") -CPU_SPECIFIC("pentium_pro", 'C', "+cmov") -CPU_SPECIFIC("pentium_mmx", 'D', "+mmx") -CPU_SPECIFIC("pentium_ii", 'E', "+cmov,+mmx") -CPU_SPECIFIC("pentium_iii", 'H', "+cmov,+mmx,+sse") -CPU_SPECIFIC_ALIAS("pentium_iii_no_xmm_regs", "pentium_iii") -CPU_SPECIFIC("pentium_4", 'J', "+cmov,+mmx,+sse,+sse2") -CPU_SPECIFIC("pentium_m", 'K', "+cmov,+mmx,+sse,+sse2") -CPU_SPECIFIC("pentium_4_sse3", 'L', "+cmov,+mmx,+sse,+sse2,+sse3") -CPU_SPECIFIC("core_2_duo_ssse3", 'M', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3") -CPU_SPECIFIC("core_2_duo_sse4_1", 'N', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1") -CPU_SPECIFIC("atom", 'O', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+movbe") -CPU_SPECIFIC("atom_sse4_2", 'c', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") -CPU_SPECIFIC("core_i7_sse4_2", 'P', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") -CPU_SPECIFIC("core_aes_pclmulqdq", 'Q', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") -CPU_SPECIFIC("atom_sse4_2_movbe", 'd', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt") -CPU_SPECIFIC("goldmont", 'i', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt") -CPU_SPECIFIC("sandybridge", 'R', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx") -CPU_SPECIFIC_ALIAS("core_2nd_gen_avx", "sandybridge") -CPU_SPECIFIC("ivybridge", 'S', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+f16c,+avx") -CPU_SPECIFIC_ALIAS("core_3rd_gen_avx", "ivybridge") -CPU_SPECIFIC("haswell", 'V', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2") -CPU_SPECIFIC_ALIAS("core_4th_gen_avx", "haswell") -CPU_SPECIFIC("core_4th_gen_avx_tsx", 'W', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2") -CPU_SPECIFIC("broadwell", 'X', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx") -CPU_SPECIFIC_ALIAS("core_5th_gen_avx", "broadwell") -CPU_SPECIFIC("core_5th_gen_avx_tsx", 'Y', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx") -CPU_SPECIFIC("knl", 'Z', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512f,+adx,+avx512er,+avx512pf,+avx512cd") -CPU_SPECIFIC_ALIAS("mic_avx512", "knl") -CPU_SPECIFIC("skylake", 'b', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx,+mpx") -CPU_SPECIFIC( "skylake_avx512", 'a', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512dq,+avx512f,+adx,+avx512cd,+avx512bw,+avx512vl,+clwb") -CPU_SPECIFIC("cannonlake", 'e', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512dq,+avx512f,+adx,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi") -CPU_SPECIFIC("knm", 'j', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512f,+adx,+avx512er,+avx512pf,+avx512cd,+avx5124fmaps,+avx5124vnniw,+avx512vpopcntdq") +CPU_SPECIFIC("generic", "generic", 'A', "") +CPU_SPECIFIC("pentium", "pentium", 'B', "") +CPU_SPECIFIC("pentium_pro", "pentiumpro", 'C', "+cmov") +CPU_SPECIFIC("pentium_mmx", "pentium-mmx", 'D', "+mmx") +CPU_SPECIFIC("pentium_ii", "pentium2", 'E', "+cmov,+mmx") +CPU_SPECIFIC("pentium_iii", "pentium3", 'H', "+cmov,+mmx,+sse") +CPU_SPECIFIC_ALIAS("pentium_iii_no_xmm_regs", "pentium3", "pentium_iii") +CPU_SPECIFIC("pentium_4", "pentium4", 'J', "+cmov,+mmx,+sse,+sse2") +CPU_SPECIFIC("pentium_m", "pentium-m", 'K', "+cmov,+mmx,+sse,+sse2") +CPU_SPECIFIC("pentium_4_sse3", "prescott", 'L', "+cmov,+mmx,+sse,+sse2,+sse3") +CPU_SPECIFIC("core_2_duo_ssse3", "core2", 'M', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3") +CPU_SPECIFIC("core_2_duo_sse4_1", "penryn", 'N', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1") +CPU_SPECIFIC("atom", "atom", 'O', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+movbe") +CPU_SPECIFIC("atom_sse4_2", "silvermont", 'c', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") +CPU_SPECIFIC("core_i7_sse4_2", "nehalem", 'P', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") +CPU_SPECIFIC("core_aes_pclmulqdq", "westmere", 'Q', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt") +CPU_SPECIFIC("atom_sse4_2_movbe", "silvermont", 'd', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt") +CPU_SPECIFIC("goldmont", "goldmont", 'i', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt") +CPU_SPECIFIC("sandybridge", "sandybridge", 'R', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx") +CPU_SPECIFIC_ALIAS("core_2nd_gen_avx", "sandybridge", "sandybridge") +CPU_SPECIFIC("ivybridge", "ivybridge", 'S', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+f16c,+avx") +CPU_SPECIFIC_ALIAS("core_3rd_gen_avx", "ivybridge", "ivybridge") +CPU_SPECIFIC("haswell", "haswell", 'V', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2") +CPU_SPECIFIC_ALIAS("core_4th_gen_avx", "haswell", "haswell") +CPU_SPECIFIC("core_4th_gen_avx_tsx", "haswell", 'W', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2") +CPU_SPECIFIC("broadwell", "broadwell", 'X', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx") +CPU_SPECIFIC_ALIAS("core_5th_gen_avx", "broadwell", "broadwell") +CPU_SPECIFIC("core_5th_gen_avx_tsx", "broadwell", 'Y', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx") +CPU_SPECIFIC("knl", "knl", 'Z', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512f,+adx,+avx512er,+avx512pf,+avx512cd") +CPU_SPECIFIC_ALIAS("mic_avx512", "knl", "knl") +CPU_SPECIFIC("skylake", "skylake", 'b', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+adx,+mpx") +CPU_SPECIFIC( "skylake_avx512", "skylake-avx512", 'a', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512dq,+avx512f,+adx,+avx512cd,+avx512bw,+avx512vl,+clwb") +CPU_SPECIFIC("cannonlake", "cannonlake", 'e', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512dq,+avx512f,+adx,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi") +CPU_SPECIFIC("knm", "knm", 'j', "+cmov,+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+movbe,+popcnt,+f16c,+avx,+fma,+bmi,+lzcnt,+avx2,+avx512f,+adx,+avx512er,+avx512pf,+avx512cd,+avx5124fmaps,+avx5124vnniw,+avx512vpopcntdq") #undef CPU_SPECIFIC_ALIAS #undef CPU_SPECIFIC diff --git a/llvm/include/llvm/Support/YAMLParser.h b/llvm/include/llvm/Support/YAMLParser.h index a4b2ab5e49ec..231cc1d28c9a 100644 --- a/llvm/include/llvm/Support/YAMLParser.h +++ b/llvm/include/llvm/Support/YAMLParser.h @@ -11,7 +11,6 @@ // See http://www.yaml.org/spec/1.2/spec.html for the full standard. // // This currently does not implement the following: -// * Multi-line literal folding. // * Tag resolution. // * UTF-16. // * BOMs anywhere other than the first Unicode scalar value in the file. diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index 7ad73543fc6e..8ade9b15642b 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -24,7 +24,6 @@ #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include <cassert> -#include <cctype> #include <map> #include <memory> #include <new> @@ -63,6 +62,7 @@ struct MappingTraits { // static void mapping(IO &io, T &fields); // Optionally may provide: // static std::string validate(IO &io, T &fields); + // static void enumInput(IO &io, T &value); // // The optional flow flag will cause generated YAML to use a flow mapping // (e.g. { a: 0, b: 1 }): @@ -446,6 +446,31 @@ template <class T> struct has_MappingValidateTraits<T, EmptyContext> { static bool const value = (sizeof(test<MappingTraits<T>>(nullptr)) == 1); }; +// Test if MappingContextTraits<T>::enumInput() is defined on type T. +template <class T, class Context> struct has_MappingEnumInputTraits { + using Signature_validate = void (*)(class IO &, T &); + + template <typename U> + static char test(SameType<Signature_validate, &U::enumInput> *); + + template <typename U> static double test(...); + + static bool const value = + (sizeof(test<MappingContextTraits<T, Context>>(nullptr)) == 1); +}; + +// Test if MappingTraits<T>::enumInput() is defined on type T. +template <class T> struct has_MappingEnumInputTraits<T, EmptyContext> { + using Signature_validate = void (*)(class IO &, T &); + + template <typename U> + static char test(SameType<Signature_validate, &U::enumInput> *); + + template <typename U> static double test(...); + + static bool const value = (sizeof(test<MappingTraits<T>>(nullptr)) == 1); +}; + // Test if SequenceTraits<T> is defined on type T. template <class T> struct has_SequenceMethodTraits @@ -537,9 +562,8 @@ template <class T> struct has_PolymorphicTraits { }; inline bool isNumeric(StringRef S) { - const static auto skipDigits = [](StringRef Input) { - return Input.drop_front( - std::min(Input.find_first_not_of("0123456789"), Input.size())); + const auto skipDigits = [](StringRef Input) { + return Input.ltrim("0123456789"); }; // Make S.front() and S.drop_front().front() (if S.front() is [+-]) calls @@ -666,8 +690,7 @@ inline QuotingType needsQuotes(StringRef S) { // 7.3.3 Plain Style // Plain scalars must not begin with most indicators, as this would cause // ambiguity with other YAML constructs. - static constexpr char Indicators[] = R"(-?:\,[]{}#&*!|>'"%@`)"; - if (S.find_first_of(Indicators) == 0) + if (std::strchr(R"(-?:\,[]{}#&*!|>'"%@`)", S[0]) != nullptr) MaxQuotingNeeded = QuotingType::Single; for (unsigned char C : S) { @@ -1062,8 +1085,29 @@ yamlize(IO &io, T &Val, bool, Context &Ctx) { } template <typename T, typename Context> +std::enable_if_t<!has_MappingEnumInputTraits<T, Context>::value, bool> +yamlizeMappingEnumInput(IO &io, T &Val) { + return false; +} + +template <typename T, typename Context> +std::enable_if_t<has_MappingEnumInputTraits<T, Context>::value, bool> +yamlizeMappingEnumInput(IO &io, T &Val) { + if (io.outputting()) + return false; + + io.beginEnumScalar(); + MappingTraits<T>::enumInput(io, Val); + bool Matched = !io.matchEnumFallback(); + io.endEnumScalar(); + return Matched; +} + +template <typename T, typename Context> std::enable_if_t<unvalidatedMappingTraits<T, Context>::value, void> yamlize(IO &io, T &Val, bool, Context &Ctx) { + if (yamlizeMappingEnumInput<T, Context>(io, Val)) + return; if (has_FlowTraits<MappingTraits<T>>::value) { io.beginFlowMapping(); detail::doMapping(io, Val, Ctx); @@ -1624,14 +1668,13 @@ template <typename T, typename Context> void IO::processKeyWithDefault(const char *Key, Optional<T> &Val, const Optional<T> &DefaultValue, bool Required, Context &Ctx) { - assert(DefaultValue.hasValue() == false && - "Optional<T> shouldn't have a value!"); + assert(!DefaultValue && "Optional<T> shouldn't have a value!"); void *SaveInfo; bool UseDefault = true; - const bool sameAsDefault = outputting() && !Val.hasValue(); - if (!outputting() && !Val.hasValue()) + const bool sameAsDefault = outputting() && !Val; + if (!outputting() && !Val) Val = T(); - if (Val.hasValue() && + if (Val && this->preflightKey(Key, Required, sameAsDefault, UseDefault, SaveInfo)) { // When reading an Optional<X> key from a YAML description, we allow the @@ -1648,7 +1691,7 @@ void IO::processKeyWithDefault(const char *Key, Optional<T> &Val, if (IsNone) Val = DefaultValue; else - yamlize(*this, Val.getValue(), Required, Ctx); + yamlize(*this, *Val, Required, Ctx); this->postflightKey(SaveInfo); } else { if (UseDefault) diff --git a/llvm/include/llvm/Support/circular_raw_ostream.h b/llvm/include/llvm/Support/circular_raw_ostream.h index d2f01ea6a7f2..17fb8fa0e476 100644 --- a/llvm/include/llvm/Support/circular_raw_ostream.h +++ b/llvm/include/llvm/Support/circular_raw_ostream.h @@ -38,7 +38,7 @@ namespace llvm { /// TheStream - The real stream we output to. We set it to be /// unbuffered, since we're already doing our own buffering. /// - raw_ostream *TheStream; + raw_ostream *TheStream = nullptr; /// OwnsStream - Are we responsible for managing the underlying /// stream? @@ -51,7 +51,7 @@ namespace llvm { /// BufferArray - The actual buffer storage. /// - char *BufferArray; + char *BufferArray = nullptr; /// Cur - Pointer to the current output point in BufferArray. /// @@ -60,7 +60,7 @@ namespace llvm { /// Filled - Indicate whether the buffer has been completely /// filled. This helps avoid garbage output. /// - bool Filled; + bool Filled = false; /// Banner - A pointer to a banner to print before dumping the /// log. @@ -106,9 +106,8 @@ namespace llvm { /// circular_raw_ostream(raw_ostream &Stream, const char *Header, size_t BuffSize = 0, bool Owns = REFERENCE_ONLY) - : raw_ostream(/*unbuffered*/ true), TheStream(nullptr), - OwnsStream(Owns), BufferSize(BuffSize), BufferArray(nullptr), - Filled(false), Banner(Header) { + : raw_ostream(/*unbuffered*/ true), OwnsStream(Owns), + BufferSize(BuffSize), Banner(Header) { if (BufferSize != 0) BufferArray = new char[BufferSize]; Cur = BufferArray; diff --git a/llvm/include/llvm/Support/raw_sha1_ostream.h b/llvm/include/llvm/Support/raw_sha1_ostream.h index 3991691796b5..299f6e6b5e88 100644 --- a/llvm/include/llvm/Support/raw_sha1_ostream.h +++ b/llvm/include/llvm/Support/raw_sha1_ostream.h @@ -30,7 +30,7 @@ class raw_sha1_ostream : public raw_ostream { public: /// Return the current SHA1 hash for the content of the stream - StringRef sha1() { + std::array<uint8_t, 20> sha1() { flush(); return State.result(); } diff --git a/llvm/include/llvm/TableGen/Parser.h b/llvm/include/llvm/TableGen/Parser.h new file mode 100644 index 000000000000..411259e4033c --- /dev/null +++ b/llvm/include/llvm/TableGen/Parser.h @@ -0,0 +1,34 @@ +//===- llvm/TableGen/Parser.h - tblgen parser entry point -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares an entry point into the tablegen parser for use by tools. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TABLEGEN_PARSER_H +#define LLVM_TABLEGEN_PARSER_H + +#include "llvm/ADT/STLExtras.h" +#include <string> +#include <vector> + +namespace llvm { +class RecordKeeper; +class SourceMgr; + +/// Parse the TableGen file defined within the main buffer of the given +/// SourceMgr. On success, populates the provided RecordKeeper with the parsed +/// records and returns false. On failure, returns true. +/// +/// NOTE: TableGen currently relies on global state within a given parser +/// invocation, so this function is not thread-safe. +bool TableGenParseFile(SourceMgr &InputSrcMgr, RecordKeeper &Records); + +} // end namespace llvm + +#endif // LLVM_TABLEGEN_PARSER_H diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h index 1157487eced3..44daad976c12 100644 --- a/llvm/include/llvm/TableGen/Record.h +++ b/llvm/include/llvm/TableGen/Record.h @@ -28,7 +28,6 @@ #include "llvm/Support/Timer.h" #include "llvm/Support/TrailingObjects.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <cstdint> @@ -40,7 +39,7 @@ namespace llvm { namespace detail { -struct RecordContext; +struct RecordKeeperImpl; } // namespace detail class ListRecTy; @@ -70,15 +69,20 @@ public: private: RecTyKind Kind; + /// The RecordKeeper that uniqued this Type. + RecordKeeper &RK; /// ListRecTy of the list that has elements of this type. ListRecTy *ListTy = nullptr; public: - RecTy(RecTyKind K) : Kind(K) {} + RecTy(RecTyKind K, RecordKeeper &RK) : Kind(K), RK(RK) {} virtual ~RecTy() = default; RecTyKind getRecTyKind() const { return Kind; } + /// Return the RecordKeeper that uniqued this Type. + RecordKeeper &getRecordKeeper() const { return RK; } + virtual std::string getAsString() const = 0; void print(raw_ostream &OS) const { OS << getAsString(); } void dump() const; @@ -102,16 +106,16 @@ inline raw_ostream &operator<<(raw_ostream &OS, const RecTy &Ty) { /// 'bit' - Represent a single bit class BitRecTy : public RecTy { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; - BitRecTy() : RecTy(BitRecTyKind) {} + BitRecTy(RecordKeeper &RK) : RecTy(BitRecTyKind, RK) {} public: static bool classof(const RecTy *RT) { return RT->getRecTyKind() == BitRecTyKind; } - static BitRecTy *get(); + static BitRecTy *get(RecordKeeper &RK); std::string getAsString() const override { return "bit"; } @@ -122,14 +126,15 @@ public: class BitsRecTy : public RecTy { unsigned Size; - explicit BitsRecTy(unsigned Sz) : RecTy(BitsRecTyKind), Size(Sz) {} + explicit BitsRecTy(RecordKeeper &RK, unsigned Sz) + : RecTy(BitsRecTyKind, RK), Size(Sz) {} public: static bool classof(const RecTy *RT) { return RT->getRecTyKind() == BitsRecTyKind; } - static BitsRecTy *get(unsigned Sz); + static BitsRecTy *get(RecordKeeper &RK, unsigned Sz); unsigned getNumBits() const { return Size; } @@ -142,16 +147,16 @@ public: /// 'int' - Represent an integer value of no particular size class IntRecTy : public RecTy { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; - IntRecTy() : RecTy(IntRecTyKind) {} + IntRecTy(RecordKeeper &RK) : RecTy(IntRecTyKind, RK) {} public: static bool classof(const RecTy *RT) { return RT->getRecTyKind() == IntRecTyKind; } - static IntRecTy *get(); + static IntRecTy *get(RecordKeeper &RK); std::string getAsString() const override { return "int"; } @@ -160,16 +165,16 @@ public: /// 'string' - Represent an string value class StringRecTy : public RecTy { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; - StringRecTy() : RecTy(StringRecTyKind) {} + StringRecTy(RecordKeeper &RK) : RecTy(StringRecTyKind, RK) {} public: static bool classof(const RecTy *RT) { return RT->getRecTyKind() == StringRecTyKind; } - static StringRecTy *get(); + static StringRecTy *get(RecordKeeper &RK); std::string getAsString() const override; @@ -183,7 +188,8 @@ class ListRecTy : public RecTy { RecTy *ElementTy; - explicit ListRecTy(RecTy *T) : RecTy(ListRecTyKind), ElementTy(T) {} + explicit ListRecTy(RecTy *T) + : RecTy(ListRecTyKind, T->getRecordKeeper()), ElementTy(T) {} public: static bool classof(const RecTy *RT) { @@ -202,16 +208,16 @@ public: /// 'dag' - Represent a dag fragment class DagRecTy : public RecTy { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; - DagRecTy() : RecTy(DagRecTyKind) {} + DagRecTy(RecordKeeper &RK) : RecTy(DagRecTyKind, RK) {} public: static bool classof(const RecTy *RT) { return RT->getRecTyKind() == DagRecTyKind; } - static DagRecTy *get(); + static DagRecTy *get(RecordKeeper &RK); std::string getAsString() const override; }; @@ -223,12 +229,12 @@ public: class RecordRecTy final : public RecTy, public FoldingSetNode, public TrailingObjects<RecordRecTy, Record *> { friend class Record; - friend detail::RecordContext; + friend detail::RecordKeeperImpl; unsigned NumClasses; - explicit RecordRecTy(unsigned Num) - : RecTy(RecordRecTyKind), NumClasses(Num) {} + explicit RecordRecTy(RecordKeeper &RK, unsigned Num) + : RecTy(RecordRecTyKind, RK), NumClasses(Num) {} public: RecordRecTy(const RecordRecTy &) = delete; @@ -242,7 +248,8 @@ public: } /// Get the record type with the given non-redundant list of superclasses. - static RecordRecTy *get(ArrayRef<Record *> Classes); + static RecordRecTy *get(RecordKeeper &RK, ArrayRef<Record *> Classes); + static RecordRecTy *get(Record *Class); void Profile(FoldingSetNodeID &ID) const; @@ -304,6 +311,7 @@ protected: IK_CondOpInit, IK_FoldOpInit, IK_IsAOpInit, + IK_ExistsOpInit, IK_AnonymousNameInit, IK_StringInit, IK_VarInit, @@ -327,6 +335,9 @@ public: /// Get the kind (type) of the value. InitKind getKind() const { return Kind; } + /// Get the record keeper that initialized this Init. + RecordKeeper &getRecordKeeper() const; + protected: explicit Init(InitKind K, uint8_t Opc = 0) : Kind(K), Opc(Opc) {} @@ -426,6 +437,9 @@ public: /// Get the type of the Init as a RecTy. RecTy *getType() const { return ValueTy; } + /// Get the record keeper that initialized this Init. + RecordKeeper &getRecordKeeper() const { return ValueTy->getRecordKeeper(); } + Init *getCastTo(RecTy *Ty) const override; Init *convertInitializerTo(RecTy *Ty) const override; @@ -440,9 +454,12 @@ public: /// '?' - Represents an uninitialized value. class UnsetInit : public Init { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; - UnsetInit() : Init(IK_UnsetInit) {} + /// The record keeper that initialized this Init. + RecordKeeper &RK; + + UnsetInit(RecordKeeper &RK) : Init(IK_UnsetInit), RK(RK) {} public: UnsetInit(const UnsetInit &) = delete; @@ -453,7 +470,10 @@ public: } /// Get the singleton unset Init. - static UnsetInit *get(); + static UnsetInit *get(RecordKeeper &RK); + + /// Get the record keeper that initialized this Init. + RecordKeeper &getRecordKeeper() const { return RK; } Init *getCastTo(RecTy *Ty) const override; Init *convertInitializerTo(RecTy *Ty) const override; @@ -473,7 +493,7 @@ public: /// 'true'/'false' - Represent a concrete initializer for a bit. class BitInit final : public TypedInit { - friend detail::RecordContext; + friend detail::RecordKeeperImpl; bool Value; @@ -487,7 +507,7 @@ public: return I->getKind() == IK_BitInit; } - static BitInit *get(bool V); + static BitInit *get(RecordKeeper &RK, bool V); bool getValue() const { return Value; } @@ -508,8 +528,8 @@ class BitsInit final : public TypedInit, public FoldingSetNode, public TrailingObjects<BitsInit, Init *> { unsigned NumBits; - BitsInit(unsigned N) - : TypedInit(IK_BitsInit, BitsRecTy::get(N)), NumBits(N) {} + BitsInit(RecordKeeper &RK, unsigned N) + : TypedInit(IK_BitsInit, BitsRecTy::get(RK, N)), NumBits(N) {} public: BitsInit(const BitsInit &) = delete; @@ -522,7 +542,7 @@ public: return I->getKind() == IK_BitsInit; } - static BitsInit *get(ArrayRef<Init *> Range); + static BitsInit *get(RecordKeeper &RK, ArrayRef<Init *> Range); void Profile(FoldingSetNodeID &ID) const; @@ -558,8 +578,8 @@ public: class IntInit : public TypedInit { int64_t Value; - explicit IntInit(int64_t V) - : TypedInit(IK_IntInit, IntRecTy::get()), Value(V) {} + explicit IntInit(RecordKeeper &RK, int64_t V) + : TypedInit(IK_IntInit, IntRecTy::get(RK)), Value(V) {} public: IntInit(const IntInit &) = delete; @@ -569,7 +589,7 @@ public: return I->getKind() == IK_IntInit; } - static IntInit *get(int64_t V); + static IntInit *get(RecordKeeper &RK, int64_t V); int64_t getValue() const { return Value; } @@ -580,7 +600,7 @@ public: std::string getAsString() const override; Init *getBit(unsigned Bit) const override { - return BitInit::get((Value & (1ULL << Bit)) != 0); + return BitInit::get(getRecordKeeper(), (Value & (1ULL << Bit)) != 0); } }; @@ -588,8 +608,8 @@ public: class AnonymousNameInit : public TypedInit { unsigned Value; - explicit AnonymousNameInit(unsigned V) - : TypedInit(IK_AnonymousNameInit, StringRecTy::get()), Value(V) {} + explicit AnonymousNameInit(RecordKeeper &RK, unsigned V) + : TypedInit(IK_AnonymousNameInit, StringRecTy::get(RK)), Value(V) {} public: AnonymousNameInit(const AnonymousNameInit &) = delete; @@ -599,7 +619,7 @@ public: return I->getKind() == IK_AnonymousNameInit; } - static AnonymousNameInit *get(unsigned); + static AnonymousNameInit *get(RecordKeeper &RK, unsigned); unsigned getValue() const { return Value; } @@ -626,8 +646,8 @@ private: StringRef Value; StringFormat Format; - explicit StringInit(StringRef V, StringFormat Fmt) - : TypedInit(IK_StringInit, StringRecTy::get()), Value(V), Format(Fmt) {} + explicit StringInit(RecordKeeper &RK, StringRef V, StringFormat Fmt) + : TypedInit(IK_StringInit, StringRecTy::get(RK)), Value(V), Format(Fmt) {} public: StringInit(const StringInit &) = delete; @@ -637,7 +657,8 @@ public: return I->getKind() == IK_StringInit; } - static StringInit *get(StringRef, StringFormat Fmt = SF_String); + static StringInit *get(RecordKeeper &RK, StringRef, + StringFormat Fmt = SF_String); static StringFormat determineFormat(StringFormat Fmt1, StringFormat Fmt2) { return (Fmt1 == SF_Code || Fmt2 == SF_Code) ? SF_Code : SF_String; @@ -678,7 +699,7 @@ public: private: explicit ListInit(unsigned N, RecTy *EltTy) - : TypedInit(IK_ListInit, ListRecTy::get(EltTy)), NumValues(N) {} + : TypedInit(IK_ListInit, ListRecTy::get(EltTy)), NumValues(N) {} public: ListInit(const ListInit &) = delete; @@ -1049,8 +1070,8 @@ private: Init *Expr; IsAOpInit(RecTy *CheckType, Init *Expr) - : TypedInit(IK_IsAOpInit, IntRecTy::get()), CheckType(CheckType), - Expr(Expr) {} + : TypedInit(IK_IsAOpInit, IntRecTy::get(CheckType->getRecordKeeper())), + CheckType(CheckType), Expr(Expr) {} public: IsAOpInit(const IsAOpInit &) = delete; @@ -1075,6 +1096,40 @@ public: std::string getAsString() const override; }; +/// !exists<type>(expr) - Dynamically determine if a record of `type` named +/// `expr` exists. +class ExistsOpInit : public TypedInit, public FoldingSetNode { +private: + RecTy *CheckType; + Init *Expr; + + ExistsOpInit(RecTy *CheckType, Init *Expr) + : TypedInit(IK_ExistsOpInit, IntRecTy::get(CheckType->getRecordKeeper())), + CheckType(CheckType), Expr(Expr) {} + +public: + ExistsOpInit(const ExistsOpInit &) = delete; + ExistsOpInit &operator=(const ExistsOpInit &) = delete; + + static bool classof(const Init *I) { return I->getKind() == IK_ExistsOpInit; } + + static ExistsOpInit *get(RecTy *CheckType, Init *Expr); + + void Profile(FoldingSetNodeID &ID) const; + + // Fold - If possible, fold this to a simpler init. Return this if not + // possible to fold. + Init *Fold(Record *CurRec, bool IsFinal = false) const; + + bool isComplete() const override { return false; } + + Init *resolveReferences(Resolver &R) const override; + + Init *getBit(unsigned Bit) const override; + + std::string getAsString() const override; +}; + /// 'Opcode' - Represent a reference to an entire variable object. class VarInit : public TypedInit { Init *VarName; @@ -1118,7 +1173,8 @@ class VarBitInit final : public TypedInit { unsigned Bit; VarBitInit(TypedInit *T, unsigned B) - : TypedInit(IK_VarBitInit, BitRecTy::get()), TI(T), Bit(B) { + : TypedInit(IK_VarBitInit, BitRecTy::get(T->getRecordKeeper())), TI(T), + Bit(B) { assert(T->getType() && (isa<IntRecTy>(T->getType()) || (isa<BitsRecTy>(T->getType()) && @@ -1223,8 +1279,7 @@ class VarDefInit final : public TypedInit, public FoldingSetNode, DefInit *Def = nullptr; // after instantiation unsigned NumArgs; - explicit VarDefInit(Record *Class, unsigned N) - : TypedInit(IK_VarDefInit, RecordRecTy::get(Class)), Class(Class), NumArgs(N) {} + explicit VarDefInit(Record *Class, unsigned N); DefInit *instantiate(); @@ -1321,8 +1376,8 @@ class DagInit final : public TypedInit, public FoldingSetNode, unsigned NumArgNames; DagInit(Init *V, StringInit *VN, unsigned NumArgs, unsigned NumArgNames) - : TypedInit(IK_DagInit, DagRecTy::get()), Val(V), ValName(VN), - NumArgs(NumArgs), NumArgNames(NumArgNames) {} + : TypedInit(IK_DagInit, DagRecTy::get(V->getRecordKeeper())), Val(V), + ValName(VN), NumArgs(NumArgs), NumArgNames(NumArgNames) {} size_t numTrailingObjects(OverloadToken<Init *>) const { return NumArgs; } @@ -1427,6 +1482,9 @@ public: RecordVal(Init *N, RecTy *T, FieldKind K); RecordVal(Init *N, SMLoc Loc, RecTy *T, FieldKind K); + /// Get the record keeper used to unique this value. + RecordKeeper &getRecordKeeper() const { return Name->getRecordKeeper(); } + /// Get the name of the field as a StringRef. StringRef getName() const; @@ -1527,13 +1585,14 @@ public: explicit Record(Init *N, ArrayRef<SMLoc> locs, RecordKeeper &records, bool Anonymous = false, bool Class = false) : Name(N), Locs(locs.begin(), locs.end()), TrackedRecords(records), - ID(getNewUID()), IsAnonymous(Anonymous), IsClass(Class) { + ID(getNewUID(N->getRecordKeeper())), IsAnonymous(Anonymous), + IsClass(Class) { checkName(); } explicit Record(StringRef N, ArrayRef<SMLoc> locs, RecordKeeper &records, bool Class = false) - : Record(StringInit::get(N), locs, records, false, Class) {} + : Record(StringInit::get(records, N), locs, records, false, Class) {} // When copy-constructing a Record, we must still guarantee a globally unique // ID number. Don't copy CorrespondingDefInit either, since it's owned by the @@ -1542,9 +1601,10 @@ public: : Name(O.Name), Locs(O.Locs), TemplateArgs(O.TemplateArgs), Values(O.Values), Assertions(O.Assertions), SuperClasses(O.SuperClasses), TrackedRecords(O.TrackedRecords), - ID(getNewUID()), IsAnonymous(O.IsAnonymous), IsClass(O.IsClass) {} + ID(getNewUID(O.getRecords())), IsAnonymous(O.IsAnonymous), + IsClass(O.IsClass) {} - static unsigned getNewUID(); + static unsigned getNewUID(RecordKeeper &RK); unsigned getID() const { return ID; } @@ -1600,7 +1660,7 @@ public: } const RecordVal *getValue(StringRef Name) const { - return getValue(StringInit::get(Name)); + return getValue(StringInit::get(getRecords(), Name)); } RecordVal *getValue(const Init *Name) { @@ -1631,7 +1691,7 @@ public: } void removeValue(StringRef Name) { - removeValue(StringInit::get(Name)); + removeValue(StringInit::get(getRecords(), Name)); } void addAssertion(SMLoc Loc, Init *Condition, Init *Message) { @@ -1671,11 +1731,11 @@ public: SuperClasses.push_back(std::make_pair(R, Range)); } - /// If there are any field references that refer to fields - /// that have been filled in, we can propagate the values now. + /// If there are any field references that refer to fields that have been + /// filled in, we can propagate the values now. /// - /// This is a final resolve: any error messages, e.g. due to undefined - /// !cast references, are generated now. + /// This is a final resolve: any error messages, e.g. due to undefined !cast + /// references, are generated now. void resolveReferences(Init *NewName = nullptr); /// Apply the resolver to the name of the record as well as to the @@ -1699,11 +1759,11 @@ public: // High-level methods useful to tablegen back-ends // - ///Return the source location for the named field. + /// Return the source location for the named field. SMLoc getFieldLoc(StringRef FieldName) const; - /// Return the initializer for a value with the specified name, - /// or throw an exception if the field does not exist. + /// Return the initializer for a value with the specified name, or throw an + /// exception if the field does not exist. Init *getValueInit(StringRef FieldName) const; /// Return true if the named field is unset. @@ -1711,96 +1771,85 @@ public: return isa<UnsetInit>(getValueInit(FieldName)); } - /// This method looks up the specified field and returns - /// its value as a string, throwing an exception if the field does not exist - /// or if the value is not a string. + /// This method looks up the specified field and returns its value as a + /// string, throwing an exception if the field does not exist or if the value + /// is not a string. StringRef getValueAsString(StringRef FieldName) const; - /// This method looks up the specified field and returns - /// its value as a string, throwing an exception if the field if the value is - /// not a string and llvm::Optional() if the field does not exist. + /// This method looks up the specified field and returns its value as a + /// string, throwing an exception if the value is not a string and + /// llvm::Optional() if the field does not exist. llvm::Optional<StringRef> getValueAsOptionalString(StringRef FieldName) const; - /// This method looks up the specified field and returns - /// its value as a BitsInit, throwing an exception if the field does not exist - /// or if the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// BitsInit, throwing an exception if the field does not exist or if the + /// value is not the right type. BitsInit *getValueAsBitsInit(StringRef FieldName) const; - /// This method looks up the specified field and returns - /// its value as a ListInit, throwing an exception if the field does not exist - /// or if the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// ListInit, throwing an exception if the field does not exist or if the + /// value is not the right type. ListInit *getValueAsListInit(StringRef FieldName) const; - /// This method looks up the specified field and - /// returns its value as a vector of records, throwing an exception if the - /// field does not exist or if the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// vector of records, throwing an exception if the field does not exist or + /// if the value is not the right type. std::vector<Record*> getValueAsListOfDefs(StringRef FieldName) const; - /// This method looks up the specified field and - /// returns its value as a vector of integers, throwing an exception if the - /// field does not exist or if the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// vector of integers, throwing an exception if the field does not exist or + /// if the value is not the right type. std::vector<int64_t> getValueAsListOfInts(StringRef FieldName) const; - /// This method looks up the specified field and - /// returns its value as a vector of strings, throwing an exception if the - /// field does not exist or if the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// vector of strings, throwing an exception if the field does not exist or + /// if the value is not the right type. std::vector<StringRef> getValueAsListOfStrings(StringRef FieldName) const; - /// This method looks up the specified field and returns its - /// value as a Record, throwing an exception if the field does not exist or if - /// the value is not the right type. + /// This method looks up the specified field and returns its value as a + /// Record, throwing an exception if the field does not exist or if the value + /// is not the right type. Record *getValueAsDef(StringRef FieldName) const; /// This method looks up the specified field and returns its value as a - /// Record, returning null if the field exists but is "uninitialized" - /// (i.e. set to `?`), and throwing an exception if the field does not - /// exist or if its value is not the right type. + /// Record, returning null if the field exists but is "uninitialized" (i.e. + /// set to `?`), and throwing an exception if the field does not exist or if + /// its value is not the right type. Record *getValueAsOptionalDef(StringRef FieldName) const; - /// This method looks up the specified field and returns its - /// value as a bit, throwing an exception if the field does not exist or if - /// the value is not the right type. + /// This method looks up the specified field and returns its value as a bit, + /// throwing an exception if the field does not exist or if the value is not + /// the right type. bool getValueAsBit(StringRef FieldName) const; - /// This method looks up the specified field and - /// returns its value as a bit. If the field is unset, sets Unset to true and - /// returns false. + /// This method looks up the specified field and returns its value as a bit. + /// If the field is unset, sets Unset to true and returns false. bool getValueAsBitOrUnset(StringRef FieldName, bool &Unset) const; - /// This method looks up the specified field and returns its - /// value as an int64_t, throwing an exception if the field does not exist or - /// if the value is not the right type. + /// This method looks up the specified field and returns its value as an + /// int64_t, throwing an exception if the field does not exist or if the + /// value is not the right type. int64_t getValueAsInt(StringRef FieldName) const; - /// This method looks up the specified field and returns its - /// value as an Dag, throwing an exception if the field does not exist or if - /// the value is not the right type. + /// This method looks up the specified field and returns its value as an Dag, + /// throwing an exception if the field does not exist or if the value is not + /// the right type. DagInit *getValueAsDag(StringRef FieldName) const; }; raw_ostream &operator<<(raw_ostream &OS, const Record &R); class RecordKeeper { - friend class RecordRecTy; - using RecordMap = std::map<std::string, std::unique_ptr<Record>, std::less<>>; using GlobalMap = std::map<std::string, Init *, std::less<>>; - std::string InputFilename; - RecordMap Classes, Defs; - mutable StringMap<std::vector<Record *>> ClassRecordsMap; - FoldingSet<RecordRecTy> RecordTypePool; - std::map<std::string, Init *, std::less<>> ExtraGlobals; - unsigned AnonCounter = 0; +public: + RecordKeeper(); + ~RecordKeeper(); - // These members are for the phase timing feature. We need a timer group, - // the last timer started, and a flag to say whether the last timer - // is the special "backend overall timer." - TimerGroup *TimingGroup = nullptr; - Timer *LastTimer = nullptr; - bool BackendTimer = false; + /// Return the internal implementation of the RecordKeeper. + detail::RecordKeeperImpl &getImpl() { return *Impl; } -public: /// Get the main TableGen input file's name. const std::string getInputFilename() const { return InputFilename; } @@ -1896,7 +1945,33 @@ public: std::vector<Record *> getAllDerivedDefinitions( ArrayRef<StringRef> ClassNames) const; + /// Get all the concrete records that inherit from specified class, if the + /// class is defined. Returns an empty vector if the class is not defined. + std::vector<Record *> + getAllDerivedDefinitionsIfDefined(StringRef ClassName) const; + void dump() const; + +private: + RecordKeeper(RecordKeeper &&) = delete; + RecordKeeper(const RecordKeeper &) = delete; + RecordKeeper &operator=(RecordKeeper &&) = delete; + RecordKeeper &operator=(const RecordKeeper &) = delete; + + std::string InputFilename; + RecordMap Classes, Defs; + mutable StringMap<std::vector<Record *>> ClassRecordsMap; + GlobalMap ExtraGlobals; + + // These members are for the phase timing feature. We need a timer group, + // the last timer started, and a flag to say whether the last timer + // is the special "backend overall timer." + TimerGroup *TimingGroup = nullptr; + Timer *LastTimer = nullptr; + bool BackendTimer = false; + + /// The internal uniquer implementation of the RecordKeeper. + std::unique_ptr<detail::RecordKeeperImpl> Impl; }; /// Sorting predicate to sort record pointers by name. diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index f84889392d13..7a6d91061701 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -42,6 +42,7 @@ struct CGPassBuilderOption { bool DisableMergeICmps = false; bool DisablePartialLibcallInlining = false; bool DisableConstantHoisting = false; + bool DisableSelectOptimize = true; bool PrintISelInput = false; bool PrintGCInfo = false; bool RequiresCodeGenSCCOrder = false; diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 2af20ab6a53f..3e2f18b57d1e 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -745,6 +745,13 @@ def G_FCANONICALIZE : GenericInstruction { let hasSideEffects = false; } +// Generic opcode equivalent to the llvm.is_fpclass intrinsic. +def G_IS_FPCLASS: GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$src, unknown:$test, unknown:$fpsem); + let hasSideEffects = false; +} + // FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two // values. // @@ -965,6 +972,12 @@ def G_FNEARBYINT : GenericInstruction { //------------------------------------------------------------------------------ // Opcodes for LLVM Intrinsics //------------------------------------------------------------------------------ +def G_INTRINSIC_FPTRUNC_ROUND : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$src1, i32imm:$round_mode); + let hasSideEffects = false; +} + def G_INTRINSIC_TRUNC : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 4859cf6b57b7..89f08d200021 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -118,6 +118,7 @@ def int64_matchinfo: GIDefMatchData<"int64_t">; def apint_matchinfo : GIDefMatchData<"APInt">; def build_fn_matchinfo : GIDefMatchData<"std::function<void(MachineIRBuilder &)>">; +def unsigned_matchinfo: GIDefMatchData<"unsigned">; def copy_prop : GICombineRule< (defs root:$d), @@ -234,6 +235,12 @@ def binop_left_undef_to_zero: GICombineRule< [{ return Helper.matchOperandIsUndef(*${root}, 1); }]), (apply [{ Helper.replaceInstWithConstant(*${root}, 0); }])>; +def binop_right_undef_to_undef: GICombineRule< + (defs root:$root), + (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR):$root, + [{ return Helper.matchOperandIsUndef(*${root}, 2); }]), + (apply [{ Helper.replaceInstWithUndef(*${root}); }])>; + // Instructions where if any source operand is undef, the instruction can be // replaced with undef. def propagate_undef_any_op: GICombineRule< @@ -283,6 +290,13 @@ def select_constant_cmp: GICombineRule< (apply [{ return Helper.replaceSingleDefInstWithOperand(*${root}, ${matchinfo}); }]) >; +def select_to_logical : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (wip_match_opcode G_SELECT):$root, + [{ return Helper.matchSelectToLogical(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }]) +>; + // Fold x op 0 -> x def right_identity_zero: GICombineRule< (defs root:$root), @@ -323,6 +337,26 @@ def urem_pow2_to_mask : GICombineRule< (apply [{ Helper.applySimplifyURemByPow2(*${root}); }]) >; +// Push a binary operator through a select on constants. +// +// binop (select cond, K0, K1), K2 -> +// select cond, (binop K0, K2), (binop K1, K2) + +// Every binary operator that has constant folding. We currently do +// not have constant folding for G_FPOW, G_FMAXNUM_IEEE or +// G_FMINNUM_IEEE. +def fold_binop_into_select : GICombineRule< + (defs root:$root, unsigned_matchinfo:$select_op_no), + (match (wip_match_opcode + G_ADD, G_SUB, G_PTR_ADD, G_AND, G_OR, G_XOR, + G_SDIV, G_SREM, G_UDIV, G_UREM, G_LSHR, G_ASHR, G_SHL, + G_SMIN, G_SMAX, G_UMIN, G_UMAX, + G_FMUL, G_FADD, G_FSUB, G_FDIV, G_FREM, + G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root, + [{ return Helper.matchFoldBinOpIntoSelect(*${root}, ${select_op_no}); }]), + (apply [{ return Helper.applyFoldBinOpIntoSelect(*${root}, ${select_op_no}); }]) +>; + // Transform d = [su]div(x, y) and r = [su]rem(x, y) - > d, r = [su]divrem(x, y) def div_rem_to_divrem_matchdata : GIDefMatchData<"MachineInstr *">; def div_rem_to_divrem : GICombineRule< @@ -753,6 +787,18 @@ def mulo_by_2: GICombineRule< [{ return Helper.matchMulOBy2(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>; +def mulo_by_0: GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (wip_match_opcode G_UMULO, G_SMULO):$root, + [{ return Helper.matchMulOBy0(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; + +def addo_by_0: GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (wip_match_opcode G_UADDO, G_SADDO):$root, + [{ return Helper.matchAddOBy0(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; + def mulh_to_lshr : GICombineRule< (defs root:$root), (match (wip_match_opcode G_UMULH):$root, @@ -845,10 +891,26 @@ def combine_fsub_fpext_fneg_fmul_to_fmad_or_fma: GICombineRule< *${root}, ${info}); }]), (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>; +def combine_minmax_nan: GICombineRule< + (defs root:$root, unsigned_matchinfo:$info), + (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root, + [{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]), + (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${info}); }])>; + +// Transform (add x, (sub y, x)) -> y +// Transform (add (sub y, x), x) -> y +def add_sub_reg: GICombineRule < + (defs root:$root, register_matchinfo:$matchinfo), + (match (wip_match_opcode G_ADD):$root, + [{ return Helper.matchAddSubSameReg(*${root}, ${matchinfo}); }]), + (apply [{ return Helper.replaceSingleDefInstWithReg(*${root}, + ${matchinfo}); }])>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, binop_left_undef_to_zero, + binop_right_undef_to_undef, propagate_undef_any_op, propagate_undef_all_ops, propagate_undef_shuffle_mask, @@ -859,10 +921,12 @@ def identity_combines : GICombineGroup<[select_same_val, right_identity_zero, binop_same_val, binop_left_to_zero, binop_right_to_zero, p2i_to_i2p, i2p_to_p2i, anyext_trunc_fold, - fneg_fneg_fold, right_identity_one]>; + fneg_fneg_fold, right_identity_one, + add_sub_reg]>; def const_combines : GICombineGroup<[constant_fp_op, const_ptradd_to_i2p, - overlapping_and, mulo_by_2]>; + overlapping_and, mulo_by_2, mulo_by_0, + addo_by_0, combine_minmax_nan]>; def known_bits_simplifications : GICombineGroup<[ redundant_and, redundant_sext_inreg, redundant_or, urem_pow2_to_mask, @@ -873,7 +937,8 @@ def width_reduction_combines : GICombineGroup<[reduce_shl_of_extend, def phi_combines : GICombineGroup<[extend_through_phis]>; -def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp]>; +def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp, + select_to_logical]>; def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl, add_p2i_to_ptradd, mul_by_neg_one]>; @@ -900,7 +965,7 @@ def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines, truncstore_merge, div_rem_to_divrem, funnel_shift_combines, form_bitfield_extract, constant_fold, fabs_fneg_fold, intdiv_combines, mulh_combines, redundant_neg_operands, - and_or_disjoint_mask, fma_combines]>; + and_or_disjoint_mask, fma_combines, fold_binop_into_select]>; // A combine group used to for prelegalizer combiners at -O0. The combines in // this group have been selected based on experiments to balance code size and diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index d8faa63ee877..c5b2462dc868 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -279,6 +279,8 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment, // heuristic. Classes with higher priority values are assigned first. This is // useful as it is sometimes beneficial to assign registers to highly // constrained classes first. The value has to be in the range [0,63]. + // Values >= 32 should be used with care since they may overlap with other + // fields in the allocator's priority heuristics. int AllocationPriority = 0; // Generate register pressure set for this register class and any class @@ -389,6 +391,14 @@ class RegisterTuples<list<SubRegIndex> Indices, list<dag> Regs, list<string> RegAsmNames = RegNames; } +// RegisterCategory - This class is a list of RegisterClasses that belong to a +// general cateogry --- e.g. "general purpose" or "fixed" registers. This is +// useful for identifying registers in a generic way instead of having +// information about a specific target's registers. +class RegisterCategory<list<RegisterClass> classes> { + // Classes - A list of register classes that fall within the category. + list<RegisterClass> Classes = classes; +} //===----------------------------------------------------------------------===// // DwarfRegNum - This class provides a mapping of the llvm register enumeration @@ -560,6 +570,9 @@ class Instruction : InstructionEncoding { bit isPseudo = false; // Is this instruction a pseudo-instruction? // If so, won't have encoding information for // the [MC]CodeEmitter stuff. + bit isMeta = false; // Is this instruction a meta-instruction? + // If so, won't produce any output in the form of + // executable instructions bit isExtractSubreg = false; // Is this instruction a kind of extract subreg? // If so, make sure to override // TargetInstrInfo::getExtractSubregLikeInputs. @@ -748,6 +761,33 @@ def ins; /// of operands. def variable_ops; +/// variable-length instruction encoding utilities. +/// The `ascend` operator should be used like this: +/// (ascend 0b0010, 0b1101) +/// Which represent a seqence of encoding fragments placing from LSB to MSB. +/// Thus, in this case the final encoding will be 0b1101_0010. +/// The arguments for `ascend` can either be `bits` or another DAG. +def ascend; +/// In addition, we can use `descend` to describe an encoding that places +/// its arguments (i.e. encoding fragments) from MSB to LSB. For instance: +/// (descend 0b0010, 0b1101) +/// This results in an encoding of 0b0010_1101. +def descend; +/// The `operand` operator should be used like this: +/// (operand "$src", 4) +/// Which represents a 4-bit encoding for an instruction operand named `$src`. +def operand; +/// Similar to `operand`, we can reference only part of the operand's encoding: +/// (slice "$src", 6, 8) +/// (slice "$src", 8, 6) +/// Both DAG represent bit 6 to 8 (total of 3 bits) in the encoding of operand +/// `$src`. +def slice; +/// You can use `encoder` to specify a custom encoder function for a specific +/// `operand` or `encoder` directive. For example: +/// (operand "$src", 4, (encoder "encodeMyImm")) +/// (slice "$src", 8, 6, (encoder "encodeMyReg")) +def encoder; /// PointerLikeRegClass - Values that are designed to have pointer width are /// derived from this. TableGen treats the register class as having a symbolic @@ -1064,6 +1104,7 @@ def CFI_INSTRUCTION : StandardPseudoInstruction { let hasCtrlDep = true; let hasSideEffects = false; let isNotDuplicable = true; + let isMeta = true; } def EH_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); @@ -1072,6 +1113,7 @@ def EH_LABEL : StandardPseudoInstruction { let hasCtrlDep = true; let hasSideEffects = false; let isNotDuplicable = true; + let isMeta = true; } def GC_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); @@ -1080,6 +1122,7 @@ def GC_LABEL : StandardPseudoInstruction { let hasCtrlDep = true; let hasSideEffects = false; let isNotDuplicable = true; + let isMeta = true; } def ANNOTATION_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); @@ -1094,6 +1137,7 @@ def KILL : StandardPseudoInstruction { let InOperandList = (ins variable_ops); let AsmString = ""; let hasSideEffects = false; + let isMeta = true; } def EXTRACT_SUBREG : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); @@ -1115,6 +1159,7 @@ def IMPLICIT_DEF : StandardPseudoInstruction { let hasSideEffects = false; let isReMaterializable = true; let isAsCheapAsAMove = true; + let isMeta = true; } def SUBREG_TO_REG : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); @@ -1134,30 +1179,35 @@ def DBG_VALUE : StandardPseudoInstruction { let InOperandList = (ins variable_ops); let AsmString = "DBG_VALUE"; let hasSideEffects = false; + let isMeta = true; } def DBG_VALUE_LIST : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "DBG_VALUE_LIST"; let hasSideEffects = 0; + let isMeta = true; } def DBG_INSTR_REF : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "DBG_INSTR_REF"; let hasSideEffects = false; + let isMeta = true; } def DBG_PHI : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "DBG_PHI"; let hasSideEffects = 0; + let isMeta = true; } def DBG_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins unknown:$label); let AsmString = "DBG_LABEL"; let hasSideEffects = false; + let isMeta = true; } def REG_SEQUENCE : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); @@ -1185,18 +1235,21 @@ def LIFETIME_START : StandardPseudoInstruction { let InOperandList = (ins i32imm:$id); let AsmString = "LIFETIME_START"; let hasSideEffects = false; + let isMeta = true; } def LIFETIME_END : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = "LIFETIME_END"; let hasSideEffects = false; + let isMeta = true; } def PSEUDO_PROBE : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i64imm:$guid, i64imm:$index, i8imm:$type, i32imm:$attr); let AsmString = "PSEUDO_PROBE"; let hasSideEffects = 1; + let isMeta = true; } def ARITH_FENCE : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); @@ -1204,6 +1257,7 @@ def ARITH_FENCE : StandardPseudoInstruction { let AsmString = ""; let hasSideEffects = false; let Constraints = "$src = $dst"; + let isMeta = true; } def STACKMAP : StandardPseudoInstruction { diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h index 392ee4334cb5..0c09cfe68478 100644 --- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -20,6 +20,7 @@ namespace llvm { +struct Align; class Constant; class DataLayout; class Function; @@ -276,7 +277,7 @@ public: } /// If supported, return the function entry point symbol. - /// Otherwise, returns nulltpr. + /// Otherwise, returns nullptr. /// Func must be a function or an alias which has a function as base object. virtual MCSymbol *getFunctionEntryPointSymbol(const GlobalValue *Func, const TargetMachine &TM) const { diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index acfb265a9ff9..bf37ad7010ec 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -18,7 +18,6 @@ #include "llvm/ADT/Triple.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/PGOOptions.h" @@ -30,8 +29,6 @@ namespace llvm { class AAManager; -template <typename IRUnitT, typename AnalysisManagerT, typename... ExtraArgTs> -class PassManager; using ModulePassManager = PassManager<Module>; class Function; @@ -225,7 +222,10 @@ public: /// Returns the code model. The choices are small, kernel, medium, large, and /// target default. - CodeModel::Model getCodeModel() const; + CodeModel::Model getCodeModel() const { return CMModel; } + + /// Set the code model. + void setCodeModel(CodeModel::Model CM) { CMModel = CM; } bool isPositionIndependent() const; @@ -260,6 +260,8 @@ public: Options.SupportsDebugEntryValues = Enable; } + void setCFIFixup(bool Enable) { Options.EnableCFIFixup = Enable; } + bool getAIXExtendedAltivecABI() const { return Options.EnableAIXExtendedAltivecABI; } @@ -337,13 +339,13 @@ public: /// This is used to construct the new pass manager's target IR analysis pass, /// set up appropriately for this target machine. Even the old pass manager /// uses this to answer queries about the IR. - TargetIRAnalysis getTargetIRAnalysis(); + TargetIRAnalysis getTargetIRAnalysis() const; /// Return a TargetTransformInfo for a given function. /// /// The returned TargetTransformInfo is specialized to the subtarget /// corresponding to \p F. - virtual TargetTransformInfo getTargetTransformInfo(const Function &F); + virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const; /// Allow the target to modify the pass manager, e.g. by calling /// PassManagerBuilder::addExtension. @@ -398,6 +400,12 @@ public: virtual unsigned getSjLjDataSize() const { return DefaultSjLjDataSize; } static std::pair<int, int> parseBinutilsVersion(StringRef Version); + + /// getAddressSpaceForPseudoSourceKind - Given the kind of memory + /// (e.g. stack) the target returns the corresponding address space. + virtual unsigned getAddressSpaceForPseudoSourceKind(unsigned Kind) const { + return 0; + } }; /// This class describes a target machine that is implemented with the LLVM @@ -417,7 +425,7 @@ public: /// /// The TTI returned uses the common code generator to answer queries about /// the IR. - TargetTransformInfo getTargetTransformInfo(const Function &F) override; + TargetTransformInfo getTargetTransformInfo(const Function &F) const override; /// Create a pass configuration object to be used by addPassToEmitX methods /// for generating a pipeline of CodeGen passes. diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index a636c4822832..6083d18d96f7 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -130,19 +130,21 @@ namespace llvm { HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false), GuaranteedTailCallOpt(false), StackSymbolOrdering(true), EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false), - DisableIntegratedAS(false), RelaxELFRelocations(false), - FunctionSections(false), DataSections(false), - IgnoreXCOFFVisibility(false), XCOFFTracebackTable(true), - UniqueSectionNames(true), UniqueBasicBlockSectionNames(false), - TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0), - EmulatedTLS(false), ExplicitEmulatedTLS(false), EnableIPRA(false), + LowerGlobalDtorsViaCxaAtExit(false), DisableIntegratedAS(false), + RelaxELFRelocations(false), FunctionSections(false), + DataSections(false), IgnoreXCOFFVisibility(false), + XCOFFTracebackTable(true), UniqueSectionNames(true), + UniqueBasicBlockSectionNames(false), TrapUnreachable(false), + NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false), + ExplicitEmulatedTLS(false), EnableIPRA(false), EmitStackSizeSection(false), EnableMachineOutliner(false), EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false), EmitAddrsig(false), EmitCallSiteInfo(false), SupportsDebugEntryValues(false), EnableDebugEntryValues(false), ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false), XRayOmitFunctionIndex(false), DebugStrictDwarf(false), - Hotpatch(false), + Hotpatch(false), PPCGenScalarMASSEntries(false), JMCInstrument(false), + EnableCFIFixup(false), MisExpect(false), FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {} /// DisableFramePointerElim - This returns true if frame pointer elimination @@ -245,6 +247,10 @@ namespace llvm { /// constructors. unsigned UseInitArray : 1; + /// Use __cxa_atexit to register global destructors; determines how + /// llvm.global_dtors is lowered. + unsigned LowerGlobalDtorsViaCxaAtExit : 1; + /// Disable the integrated assembler. unsigned DisableIntegratedAS : 1; @@ -345,6 +351,19 @@ namespace llvm { /// Emit the hotpatch flag in CodeView debug. unsigned Hotpatch : 1; + /// Enables scalar MASS conversions + unsigned PPCGenScalarMASSEntries : 1; + + /// Enable JustMyCode instrumentation. + unsigned JMCInstrument : 1; + + /// Enable the CFIFixup pass. + unsigned EnableCFIFixup : 1; + + /// When set to true, enable MisExpect Diagnostics + /// By default, it is set to false + unsigned MisExpect : 1; + /// Name of the stack usage file (i.e., .su file) if user passes /// -fstack-usage. If empty, it can be implied that -fstack-usage is not /// passed on the command line. diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index d8ef7c49a5f9..47b686aca7b5 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -238,6 +238,16 @@ def SDTMaskedLoad: SDTypeProfile<1, 4, [ // masked load SDTCisSameNumEltsAs<0, 3> ]>; +def SDTMaskedGather : SDTypeProfile<1, 4, [ + SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVec<2>, SDTCisPtrTy<3>, SDTCisVec<4>, + SDTCisSameNumEltsAs<0, 2>, SDTCisSameNumEltsAs<0, 4> +]>; + +def SDTMaskedScatter : SDTypeProfile<0, 4, [ + SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>, + SDTCisSameNumEltsAs<0, 1>, SDTCisSameNumEltsAs<0, 3> +]>; + def SDTVecShuffle : SDTypeProfile<1, 2, [ SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2> ]>; @@ -365,6 +375,10 @@ def mul : SDNode<"ISD::MUL" , SDTIntBinOp, [SDNPCommutative, SDNPAssociative]>; def mulhs : SDNode<"ISD::MULHS" , SDTIntBinOp, [SDNPCommutative]>; def mulhu : SDNode<"ISD::MULHU" , SDTIntBinOp, [SDNPCommutative]>; +def avgfloors : SDNode<"ISD::AVGFLOORS" , SDTIntBinOp, [SDNPCommutative]>; +def avgflooru : SDNode<"ISD::AVGFLOORU" , SDTIntBinOp, [SDNPCommutative]>; +def avgceils : SDNode<"ISD::AVGCEILS" , SDTIntBinOp, [SDNPCommutative]>; +def avgceilu : SDNode<"ISD::AVGCEILU" , SDTIntBinOp, [SDNPCommutative]>; def abds : SDNode<"ISD::ABDS" , SDTIntBinOp, [SDNPCommutative]>; def abdu : SDNode<"ISD::ABDU" , SDTIntBinOp, [SDNPCommutative]>; def smullohi : SDNode<"ISD::SMUL_LOHI" , SDTIntBinHiLoOp, [SDNPCommutative]>; @@ -648,6 +662,12 @@ def masked_st : SDNode<"ISD::MSTORE", SDTMaskedStore, def masked_ld : SDNode<"ISD::MLOAD", SDTMaskedLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; +def masked_gather : SDNode<"ISD::MGATHER", SDTMaskedGather, + [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; + +def masked_scatter : SDNode<"ISD::MSCATTER", SDTMaskedScatter, + [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; + // Do not use ld, st directly. Use load, extload, sextload, zextload, store, // and truncst (see below). def ld : SDNode<"ISD::LOAD" , SDTLoad, @@ -1624,6 +1644,124 @@ def atomic_load_64 : let MemoryVT = i64; } +def nonext_masked_gather : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + return cast<MaskedGatherSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; +}]>; + +// Any extending masked gather fragments. +def ext_masked_gather_i8 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::EXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def ext_masked_gather_i16 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::EXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i16; +}]>; +def ext_masked_gather_i32 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::EXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i32; +}]>; + +// Sign extending masked gather fragments. +def sext_masked_gather_i8 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::SEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def sext_masked_gather_i16 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::SEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i16; +}]>; +def sext_masked_gather_i32 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::SEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i32; +}]>; + +// Zero extending masked gather fragments. +def zext_masked_gather_i8 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::ZEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def zext_masked_gather_i16 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::ZEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i16; +}]>; +def zext_masked_gather_i32 : + PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx), + (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{ + auto MGN = cast<MaskedGatherSDNode>(N); + return MGN->getExtensionType() == ISD::ZEXTLOAD && + MGN->getMemoryVT().getScalarType() == MVT::i32; +}]>; + +// Any/Zero extending masked gather fragments. +def azext_masked_gather_i8 : + PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx), + [(ext_masked_gather_i8 node:$def, node:$pred, node:$ptr, node:$idx), + (zext_masked_gather_i8 node:$def, node:$pred, node:$ptr, node:$idx)]>; +def azext_masked_gather_i16 : + PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx), + [(ext_masked_gather_i16 node:$def, node:$pred, node:$ptr, node:$idx), + (zext_masked_gather_i16 node:$def, node:$pred, node:$ptr, node:$idx)]>; +def azext_masked_gather_i32 : + PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx), + [(ext_masked_gather_i32 node:$def, node:$pred, node:$ptr, node:$idx), + (zext_masked_gather_i32 node:$def, node:$pred, node:$ptr, node:$idx)]>; + +def nontrunc_masked_scatter : + PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx), + (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{ + return !cast<MaskedScatterSDNode>(N)->isTruncatingStore(); +}]>; + +// Truncating masked scatter fragments. +def trunc_masked_scatter_i8 : + PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx), + (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{ + auto MSN = cast<MaskedScatterSDNode>(N); + return MSN->isTruncatingStore() && + MSN->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def trunc_masked_scatter_i16 : + PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx), + (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{ + auto MSN = cast<MaskedScatterSDNode>(N); + return MSN->isTruncatingStore() && + MSN->getMemoryVT().getScalarType() == MVT::i16; +}]>; +def trunc_masked_scatter_i32 : + PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx), + (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{ + auto MSN = cast<MaskedScatterSDNode>(N); + return MSN->isTruncatingStore() && + MSN->getMemoryVT().getScalarType() == MVT::i32; +}]>; + //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. // diff --git a/llvm/include/llvm/Testing/Support/SupportHelpers.h b/llvm/include/llvm/Testing/Support/SupportHelpers.h index 2419fc95d817..b1c59cf97f7f 100644 --- a/llvm/include/llvm/Testing/Support/SupportHelpers.h +++ b/llvm/include/llvm/Testing/Support/SupportHelpers.h @@ -77,7 +77,7 @@ public: bool MatchAndExplain(const llvm::Optional<T> &Input, testing::MatchResultListener *L) const override { - return Input && ValueMatcher.MatchAndExplain(Input.getValue(), L); + return Input && ValueMatcher.MatchAndExplain(*Input, L); } void DescribeTo(std::ostream *OS) const override { @@ -238,6 +238,12 @@ public: } } + TempFile(const TempFile &) = delete; + TempFile &operator=(const TempFile &) = delete; + + TempFile(TempFile &&) = default; + TempFile &operator=(TempFile &&) = default; + /// The path to the file. StringRef path() const { return Path; } }; diff --git a/llvm/include/llvm/TextAPI/Symbol.h b/llvm/include/llvm/TextAPI/Symbol.h index dfc84908bba2..1c25295b299d 100644 --- a/llvm/include/llvm/TextAPI/Symbol.h +++ b/llvm/include/llvm/TextAPI/Symbol.h @@ -11,7 +11,6 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Error.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TextAPI/ArchitectureSet.h" #include "llvm/TextAPI/Target.h" diff --git a/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h b/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h index 072ccf7320e8..3931c9c55c07 100644 --- a/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h +++ b/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h @@ -18,10 +18,12 @@ #define LLVM_TRANSFORMS_AGGRESSIVEINSTCOMBINE_AGGRESSIVEINSTCOMBINE_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Function; +class FunctionPass; + class AggressiveInstCombinePass : public PassInfoMixin<AggressiveInstCombinePass> { diff --git a/llvm/include/llvm/Transforms/Coroutines.h b/llvm/include/llvm/Transforms/Coroutines.h deleted file mode 100644 index f68ef705fdef..000000000000 --- a/llvm/include/llvm/Transforms/Coroutines.h +++ /dev/null @@ -1,37 +0,0 @@ -//===-- Coroutines.h - Coroutine Transformations ----------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// Declare accessor functions for coroutine lowering passes. -//===----------------------------------------------------------------------===// - -#ifndef LLVM_TRANSFORMS_COROUTINES_H -#define LLVM_TRANSFORMS_COROUTINES_H - -namespace llvm { - -class Pass; -class PassManagerBuilder; - -/// Add all coroutine passes to appropriate extension points. -void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder); - -/// Lower coroutine intrinsics that are not needed by later passes. -Pass *createCoroEarlyLegacyPass(); - -/// Split up coroutines into multiple functions driving their state machines. -Pass *createCoroSplitLegacyPass(bool IsOptimizing = false); - -/// Analyze coroutines use sites, devirtualize resume/destroy calls and elide -/// heap allocation for coroutine frame where possible. -Pass *createCoroElideLegacyPass(); - -/// Lower all remaining coroutine intrinsics. -Pass *createCoroCleanupLegacyPass(); - -} - -#endif diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroCleanup.h b/llvm/include/llvm/Transforms/Coroutines/CoroCleanup.h index 7ecdc050335d..3000a38258f4 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroCleanup.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroCleanup.h @@ -18,10 +18,10 @@ namespace llvm { -class Function; +class Module; struct CoroCleanupPass : PassInfoMixin<CoroCleanupPass> { - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); static bool isRequired() { return true; } }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroConditionalWrapper.h b/llvm/include/llvm/Transforms/Coroutines/CoroConditionalWrapper.h new file mode 100644 index 000000000000..ea19ec533c4d --- /dev/null +++ b/llvm/include/llvm/Transforms/Coroutines/CoroConditionalWrapper.h @@ -0,0 +1,30 @@ +//===---- CoroConditionalWrapper.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_COROUTINES_COROCONDITIONALWRAPPER_H +#define LLVM_TRANSFORMS_COROUTINES_COROCONDITIONALWRAPPER_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Module; + +// Only runs passes in the contained pass manager if the module contains any +// coroutine intrinsic declarations. +struct CoroConditionalWrapper : PassInfoMixin<CoroConditionalWrapper> { + CoroConditionalWrapper(ModulePassManager &&); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + static bool isRequired() { return true; } + +private: + ModulePassManager PM; +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_COROUTINES_COROCONDITIONALWRAPPER_H diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroEarly.h b/llvm/include/llvm/Transforms/Coroutines/CoroEarly.h index 3f5ec2abd172..d55dcc6dfa6d 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroEarly.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroEarly.h @@ -21,10 +21,10 @@ namespace llvm { -class Function; +class Module; struct CoroEarlyPass : PassInfoMixin<CoroEarlyPass> { - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); static bool isRequired() { return true; } }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h index 67b9a93c47b2..6b7d4f4821f0 100644 --- a/llvm/include/llvm/Transforms/IPO.h +++ b/llvm/include/llvm/Transforms/IPO.h @@ -152,13 +152,6 @@ ModulePass *createDeadArgEliminationPass(); ModulePass *createDeadArgHackingPass(); //===----------------------------------------------------------------------===// -/// createArgumentPromotionPass - This pass promotes "by reference" arguments to -/// be passed by value if the number of elements passed is smaller or -/// equal to maxElements (maxElements == 0 means always promote). -/// -Pass *createArgumentPromotionPass(unsigned maxElements = 3); - -//===----------------------------------------------------------------------===// /// createOpenMPOptLegacyPass - OpenMP specific optimizations. Pass *createOpenMPOptCGSCCLegacyPass(); diff --git a/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h b/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h index 78b2f909f1c9..252cfd4dc5f3 100644 --- a/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h +++ b/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h @@ -15,10 +15,12 @@ #define LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Module; +class Pass; + /// Inlines functions marked as "always_inline". /// /// Note that this does not inline call sites marked as always_inline and does diff --git a/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h b/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h index 225def99678a..3865f098b8de 100644 --- a/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h +++ b/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h @@ -14,7 +14,6 @@ #include "llvm/IR/PassManager.h" namespace llvm { -class TargetTransformInfo; /// Argument promotion pass. /// @@ -25,10 +24,7 @@ class ArgumentPromotionPass : public PassInfoMixin<ArgumentPromotionPass> { unsigned MaxElements; public: - ArgumentPromotionPass(unsigned MaxElements = 3u) : MaxElements(MaxElements) {} - - /// Checks if a type could have padding bytes. - static bool isDenselyPacked(Type *type, const DataLayout &DL); + ArgumentPromotionPass(unsigned MaxElements = 2u) : MaxElements(MaxElements) {} PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR); diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h index 7eee16f71d64..17e29695ab73 100644 --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -116,15 +116,24 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/AbstractCallSite.h" #include "llvm/IR/ConstantRange.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/InstIterator.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Transforms/Utils/CallGraphUpdater.h" +#include <map> + namespace llvm { +class DataLayout; +class LLVMContext; +class Pass; +template <typename Fn> class function_ref; struct AADepGraphNode; struct AADepGraph; struct Attributor; @@ -140,6 +149,24 @@ class Function; /// Abstract Attribute helper functions. namespace AA { +/// Flags to distinguish intra-procedural queries from *potentially* +/// inter-procedural queries. Not that information can be valid for both and +/// therefore both bits might be set. +enum ValueScope : uint8_t { + Intraprocedural = 1, + Interprocedural = 2, +}; + +struct ValueAndContext : public std::pair<Value *, const Instruction *> { + using Base = std::pair<Value *, const Instruction *>; + ValueAndContext(const Base &B) : Base(B) {} + ValueAndContext(Value &V, const Instruction *CtxI) : Base(&V, CtxI) {} + ValueAndContext(Value &V, const Instruction &CtxI) : Base(&V, &CtxI) {} + + Value *getValue() const { return this->first; } + const Instruction *getCtxI() const { return this->second; } +}; + /// Return true if \p I is a `nosync` instruction. Use generic reasoning and /// potentially the corresponding AANoSync. bool isNoSyncInst(Attributor &A, const Instruction &I, @@ -147,18 +174,20 @@ bool isNoSyncInst(Attributor &A, const Instruction &I, /// Return true if \p V is dynamically unique, that is, there are no two /// "instances" of \p V at runtime with different values. +/// Note: If \p ForAnalysisOnly is set we only check that the Attributor will +/// never use \p V to represent two "instances" not that \p V could not +/// technically represent them. bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA, - const Value &V); + const Value &V, bool ForAnalysisOnly = true); /// Return true if \p V is a valid value in \p Scope, that is a constant or an /// instruction/argument of \p Scope. bool isValidInScope(const Value &V, const Function *Scope); -/// Return true if \p V is a valid value at position \p CtxI, that is a -/// constant, an argument of the same function as \p CtxI, or an instruction in -/// that function that dominates \p CtxI. -bool isValidAtPosition(const Value &V, const Instruction &CtxI, - InformationCache &InfoCache); +/// Return true if the value of \p VAC is a valid at the position of \p VAC, +/// that is a constant, an argument of the same function, or an instruction in +/// that function that dominates the position. +bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache); /// Try to convert \p V to type \p Ty without introducing new instructions. If /// this is not possible return `nullptr`. Note: this function basically knows @@ -192,11 +221,29 @@ bool getAssumedUnderlyingObjects(Attributor &A, const Value &Ptr, SmallVectorImpl<Value *> &Objects, const AbstractAttribute &QueryingAA, const Instruction *CtxI, - bool Intraprocedural = false); + bool &UsedAssumedInformation, + AA::ValueScope VS = Interprocedural); + +/// Collect all potential values \p LI could read into \p PotentialValues. That +/// is, the only values read by \p LI are assumed to be known and all are in +/// \p PotentialValues. \p PotentialValueOrigins will contain all the +/// instructions that might have put a potential value into \p PotentialValues. +/// Dependences onto \p QueryingAA are properly tracked, \p +/// UsedAssumedInformation will inform the caller if assumed information was +/// used. +/// +/// \returns True if the assumed potential copies are all in \p PotentialValues, +/// false if something went wrong and the copies could not be +/// determined. +bool getPotentiallyLoadedValues( + Attributor &A, LoadInst &LI, SmallSetVector<Value *, 4> &PotentialValues, + SmallSetVector<Instruction *, 4> &PotentialValueOrigins, + const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, + bool OnlyExact = false); /// Collect all potential values of the one stored by \p SI into /// \p PotentialCopies. That is, the only copies that were made via the -/// store are assumed to be known and all in \p PotentialCopies. Dependences +/// store are assumed to be known and all are in \p PotentialCopies. Dependences /// onto \p QueryingAA are properly tracked, \p UsedAssumedInformation will /// inform the caller if assumed information was used. /// @@ -205,7 +252,8 @@ bool getAssumedUnderlyingObjects(Attributor &A, const Value &Ptr, /// determined. bool getPotentialCopiesOfStoredValue( Attributor &A, StoreInst &SI, SmallSetVector<Value *, 4> &PotentialCopies, - const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation); + const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, + bool OnlyExact = false); /// Return true if \p IRP is readonly. This will query respective AAs that /// deduce the information and introduce dependences for \p QueryingAA. @@ -237,6 +285,26 @@ bool isPotentiallyReachable( } // namespace AA +template <> +struct DenseMapInfo<AA::ValueAndContext> + : public DenseMapInfo<AA::ValueAndContext::Base> { + using Base = DenseMapInfo<AA::ValueAndContext::Base>; + static inline AA::ValueAndContext getEmptyKey() { + return Base::getEmptyKey(); + } + static inline AA::ValueAndContext getTombstoneKey() { + return Base::getTombstoneKey(); + } + static unsigned getHashValue(const AA::ValueAndContext &VAC) { + return Base::getHashValue(VAC); + } + + static bool isEqual(const AA::ValueAndContext &LHS, + const AA::ValueAndContext &RHS) { + return Base::isEqual(LHS, RHS); + } +}; + /// The value passed to the line option that defines the maximal initialization /// chain length. extern unsigned MaxInitializationChainLength; @@ -1033,6 +1101,10 @@ struct InformationCache { return FI.CalledViaMustTail || FI.ContainsMustTailCall; } + bool isOnlyUsedByAssume(const Instruction &I) const { + return AssumeOnlyValues.contains(&I); + } + /// Return the analysis result from a pass \p AP for function \p F. template <typename AP> typename AP::Result *getAnalysisResultForFunction(const Function &F) { @@ -1125,6 +1197,9 @@ private: /// A map with knowledge retained in `llvm.assume` instructions. RetainedKnowledgeMap KnowledgeMap; + /// A container for all instructions that are only used by `llvm.assume`. + SetVector<const Instruction *> AssumeOnlyValues; + /// Getters for analysis. AnalysisGetter &AG; @@ -1143,6 +1218,53 @@ private: friend struct Attributor; }; +/// Configuration for the Attributor. +struct AttributorConfig { + + AttributorConfig(CallGraphUpdater &CGUpdater) : CGUpdater(CGUpdater) {} + + /// Is the user of the Attributor a module pass or not. This determines what + /// IR we can look at and modify. If it is a module pass we might deduce facts + /// outside the initial function set and modify functions outside that set, + /// but only as part of the optimization of the functions in the initial + /// function set. For CGSCC passes we can look at the IR of the module slice + /// but never run any deduction, or perform any modification, outside the + /// initial function set (which we assume is the SCC). + bool IsModulePass = true; + + /// Flag to determine if we can delete functions or keep dead ones around. + bool DeleteFns = true; + + /// Flag to determine if we rewrite function signatures. + bool RewriteSignatures = true; + + /// Flag to determine if we want to initialize all default AAs for an internal + /// function marked live. + /// TODO: This should probably be a callback, or maybe + /// identifyDefaultAbstractAttributes should be virtual, something to allow + /// customizable lazy initialization for internal functions. + bool DefaultInitializeLiveInternals = true; + + /// Helper to update an underlying call graph and to delete functions. + CallGraphUpdater &CGUpdater; + + /// If not null, a set limiting the attribute opportunities. + DenseSet<const char *> *Allowed = nullptr; + + /// Maximum number of iterations to run until fixpoint. + Optional<unsigned> MaxFixpointIterations = None; + + /// A callback function that returns an ORE object from a Function pointer. + ///{ + using OptimizationRemarkGetter = + function_ref<OptimizationRemarkEmitter &(Function *)>; + OptimizationRemarkGetter OREGetter = nullptr; + ///} + + /// The name of the pass running the attributor, used to emit remarks. + const char *PassName = nullptr; +}; + /// The fixpoint analysis framework that orchestrates the attribute deduction. /// /// The Attributor provides a general abstract analysis framework (guided @@ -1172,52 +1294,17 @@ private: /// described in the file comment. struct Attributor { - using OptimizationRemarkGetter = - function_ref<OptimizationRemarkEmitter &(Function *)>; - /// Constructor /// /// \param Functions The set of functions we are deriving attributes for. /// \param InfoCache Cache to hold various information accessible for /// the abstract attributes. - /// \param CGUpdater Helper to update an underlying call graph. - /// \param Allowed If not null, a set limiting the attribute opportunities. - /// \param DeleteFns Whether to delete functions. - /// \param RewriteSignatures Whether to rewrite function signatures. + /// \param Configuration The Attributor configuration which determines what + /// generic features to use. Attributor(SetVector<Function *> &Functions, InformationCache &InfoCache, - CallGraphUpdater &CGUpdater, - DenseSet<const char *> *Allowed = nullptr, bool DeleteFns = true, - bool RewriteSignatures = true) + AttributorConfig Configuration) : Allocator(InfoCache.Allocator), Functions(Functions), - InfoCache(InfoCache), CGUpdater(CGUpdater), Allowed(Allowed), - DeleteFns(DeleteFns), RewriteSignatures(RewriteSignatures), - MaxFixpointIterations(None), OREGetter(None), PassName("") {} - - /// Constructor - /// - /// \param Functions The set of functions we are deriving attributes for. - /// \param InfoCache Cache to hold various information accessible for - /// the abstract attributes. - /// \param CGUpdater Helper to update an underlying call graph. - /// \param Allowed If not null, a set limiting the attribute opportunities. - /// \param DeleteFns Whether to delete functions - /// \param RewriteSignatures Whether to rewrite function signatures. - /// \param MaxFixpointIterations Maximum number of iterations to run until - /// fixpoint. - /// \param OREGetter A callback function that returns an ORE object from a - /// Function pointer. - /// \param PassName The name of the pass emitting remarks. - Attributor(SetVector<Function *> &Functions, InformationCache &InfoCache, - CallGraphUpdater &CGUpdater, DenseSet<const char *> *Allowed, - bool DeleteFns, bool RewriteSignatures, - Optional<unsigned> MaxFixpointIterations, - OptimizationRemarkGetter OREGetter, const char *PassName) - : Allocator(InfoCache.Allocator), Functions(Functions), - InfoCache(InfoCache), CGUpdater(CGUpdater), Allowed(Allowed), - DeleteFns(DeleteFns), RewriteSignatures(RewriteSignatures), - MaxFixpointIterations(MaxFixpointIterations), - OREGetter(Optional<OptimizationRemarkGetter>(OREGetter)), - PassName(PassName) {} + InfoCache(InfoCache), Configuration(Configuration) {} ~Attributor(); @@ -1301,11 +1388,15 @@ struct Attributor { registerAA(AA); // For now we ignore naked and optnone functions. - bool Invalidate = Allowed && !Allowed->count(&AAType::ID); - const Function *FnScope = IRP.getAnchorScope(); - if (FnScope) - Invalidate |= FnScope->hasFnAttribute(Attribute::Naked) || - FnScope->hasFnAttribute(Attribute::OptimizeNone); + bool Invalidate = + Configuration.Allowed && !Configuration.Allowed->count(&AAType::ID); + const Function *AnchorFn = IRP.getAnchorScope(); + if (AnchorFn) { + Invalidate |= + AnchorFn->hasFnAttribute(Attribute::Naked) || + AnchorFn->hasFnAttribute(Attribute::OptimizeNone) || + (!isModulePass() && !getInfoCache().isInModuleSlice(*AnchorFn)); + } // Avoid too many nested initializations to prevent a stack overflow. Invalidate |= InitializationChainLength > MaxInitializationChainLength; @@ -1325,15 +1416,12 @@ struct Attributor { --InitializationChainLength; } - // Initialize and update is allowed for code outside of the current function - // set, but only if it is part of module slice we are allowed to look at. - // Only exception is AAIsDeadFunction whose initialization is prevented - // directly, since we don't to compute it twice. - if (FnScope && !Functions.count(const_cast<Function *>(FnScope))) { - if (!getInfoCache().isInModuleSlice(*FnScope)) { - AA.getState().indicatePessimisticFixpoint(); - return AA; - } + // We update only AAs associated with functions in the Functions set or + // call sites of them. + if ((AnchorFn && !Functions.count(const_cast<Function *>(AnchorFn))) && + !Functions.count(IRP.getAssociatedFunction())) { + AA.getState().indicatePessimisticFixpoint(); + return AA; } // If this is queried in the manifest stage, we force the AA to indicate @@ -1443,10 +1531,7 @@ struct Attributor { InformationCache &getInfoCache() { return InfoCache; } /// Return true if this is a module pass, false otherwise. - bool isModulePass() const { - return !Functions.empty() && - Functions.size() == Functions.front()->getParent()->size(); - } + bool isModulePass() const { return Configuration.IsModulePass; } /// Return true if we derive attributes for \p Fn bool isRunOn(Function &Fn) const { @@ -1481,7 +1566,8 @@ struct Attributor { assert(F.hasLocalLinkage() && "Only local linkage is assumed dead initially."); - identifyDefaultAbstractAttributes(const_cast<Function &>(F)); + if (Configuration.DefaultInitializeLiveInternals) + identifyDefaultAbstractAttributes(const_cast<Function &>(F)); } /// Helper function to remove callsite. @@ -1489,7 +1575,7 @@ struct Attributor { if (!CI) return; - CGUpdater.removeCallSite(*CI); + Configuration.CGUpdater.removeCallSite(*CI); } /// Record that \p U is to be replaces with \p NV after information was @@ -1505,11 +1591,17 @@ struct Attributor { return true; } - /// Helper function to replace all uses of \p V with \p NV. Return true if - /// there is any change. The flag \p ChangeDroppable indicates if dropppable - /// uses should be changed too. - bool changeValueAfterManifest(Value &V, Value &NV, - bool ChangeDroppable = true) { + /// Helper function to replace all uses associated with \p IRP with \p NV. + /// Return true if there is any change. The flag \p ChangeDroppable indicates + /// if dropppable uses should be changed too. + bool changeAfterManifest(const IRPosition IRP, Value &NV, + bool ChangeDroppable = true) { + if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT) { + auto *CB = cast<CallBase>(IRP.getCtxI()); + return changeUseAfterManifest( + CB->getArgOperandUse(IRP.getCallSiteArgNo()), NV); + } + Value &V = IRP.getAssociatedValue(); auto &Entry = ToBeChangedValues[&V]; Value *&CurNV = Entry.first; if (CurNV && (CurNV->stripPointerCasts() == NV.stripPointerCasts() || @@ -1532,7 +1624,7 @@ struct Attributor { /// is used, e.g., to replace \p II with a call, after information was /// manifested. void registerInvokeWithDeadSuccessor(InvokeInst &II) { - InvokeWithDeadSuccessor.push_back(&II); + InvokeWithDeadSuccessor.insert(&II); } /// Record that \p I is deleted after information was manifested. This also @@ -1551,7 +1643,9 @@ struct Attributor { /// Record that \p F is deleted after information was manifested. void deleteAfterManifest(Function &F) { - if (DeleteFns) + errs() << "Delete " << F.getName() << " : " << (Configuration.DeleteFns) + << "\n"; + if (Configuration.DeleteFns) ToBeDeletedFunctions.insert(&F); } @@ -1668,6 +1762,7 @@ public: const AbstractAttribute &QueryingAA, const Value &V, bool CheckBBLivenessOnly = false, DepClassTy LivenessDepClass = DepClassTy::OPTIONAL, + bool IgnoreDroppableUses = true, function_ref<bool(const Use &OldU, const Use &NewU)> EquivalentUseCB = nullptr); @@ -1685,37 +1780,41 @@ public: template <typename RemarkKind, typename RemarkCallBack> void emitRemark(Instruction *I, StringRef RemarkName, RemarkCallBack &&RemarkCB) const { - if (!OREGetter) + if (!Configuration.OREGetter) return; Function *F = I->getFunction(); - auto &ORE = OREGetter.getValue()(F); + auto &ORE = Configuration.OREGetter(F); if (RemarkName.startswith("OMP")) ORE.emit([&]() { - return RemarkCB(RemarkKind(PassName, RemarkName, I)) + return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I)) << " [" << RemarkName << "]"; }); else - ORE.emit([&]() { return RemarkCB(RemarkKind(PassName, RemarkName, I)); }); + ORE.emit([&]() { + return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I)); + }); } /// Emit a remark on a function. template <typename RemarkKind, typename RemarkCallBack> void emitRemark(Function *F, StringRef RemarkName, RemarkCallBack &&RemarkCB) const { - if (!OREGetter) + if (!Configuration.OREGetter) return; - auto &ORE = OREGetter.getValue()(F); + auto &ORE = Configuration.OREGetter(F); if (RemarkName.startswith("OMP")) ORE.emit([&]() { - return RemarkCB(RemarkKind(PassName, RemarkName, F)) + return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F)) << " [" << RemarkName << "]"; }); else - ORE.emit([&]() { return RemarkCB(RemarkKind(PassName, RemarkName, F)); }); + ORE.emit([&]() { + return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F)); + }); } /// Helper struct used in the communication between an abstract attribute (AA) @@ -1824,23 +1923,24 @@ public: /// This method will evaluate \p Pred on call sites and return /// true if \p Pred holds in every call sites. However, this is only possible /// all call sites are known, hence the function has internal linkage. - /// If true is returned, \p AllCallSitesKnown is set if all possible call - /// sites of the function have been visited. + /// If true is returned, \p UsedAssumedInformation is set if assumed + /// information was used to skip or simplify potential call sites. bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred, const AbstractAttribute &QueryingAA, - bool RequireAllCallSites, bool &AllCallSitesKnown); + bool RequireAllCallSites, + bool &UsedAssumedInformation); /// Check \p Pred on all call sites of \p Fn. /// /// This method will evaluate \p Pred on call sites and return /// true if \p Pred holds in every call sites. However, this is only possible /// all call sites are known, hence the function has internal linkage. - /// If true is returned, \p AllCallSitesKnown is set if all possible call - /// sites of the function have been visited. + /// If true is returned, \p UsedAssumedInformation is set if assumed + /// information was used to skip or simplify potential call sites. bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred, const Function &Fn, bool RequireAllCallSites, const AbstractAttribute *QueryingAA, - bool &AllCallSitesKnown); + bool &UsedAssumedInformation); /// Check \p Pred on all values potentially returned by \p F. /// @@ -1859,6 +1959,19 @@ public: bool checkForAllReturnedValues(function_ref<bool(Value &)> Pred, const AbstractAttribute &QueryingAA); + /// Check \p Pred on all instructions in \p Fn with an opcode present in + /// \p Opcodes. + /// + /// This method will evaluate \p Pred on all instructions with an opcode + /// present in \p Opcode and return true if \p Pred holds on all of them. + bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred, + const Function *Fn, + const AbstractAttribute &QueryingAA, + const ArrayRef<unsigned> &Opcodes, + bool &UsedAssumedInformation, + bool CheckBBLivenessOnly = false, + bool CheckPotentiallyDead = false); + /// Check \p Pred on all instructions with an opcode present in \p Opcodes. /// /// This method will evaluate \p Pred on all instructions with an opcode @@ -1987,7 +2100,7 @@ private: /// (\see registerFunctionSignatureRewrite) and return Changed if the module /// was altered. ChangeStatus - rewriteFunctionSignatures(SmallPtrSetImpl<Function *> &ModifiedFns); + rewriteFunctionSignatures(SmallSetVector<Function *, 8> &ModifiedFns); /// Check if the Attribute \p AA should be seeded. /// See getOrCreateAAFor. @@ -2011,15 +2124,12 @@ private: /// The information cache that holds pre-processed (LLVM-IR) information. InformationCache &InfoCache; - /// Helper to update an underlying call graph. - CallGraphUpdater &CGUpdater; - /// Abstract Attribute dependency graph AADepGraph DG; /// Set of functions for which we modified the content such that it might /// impact the call graph. - SmallPtrSet<Function *, 8> CGModifiedFunctions; + SmallSetVector<Function *, 8> CGModifiedFunctions; /// Information about a dependence. If FromAA is changed ToAA needs to be /// updated as well. @@ -2039,34 +2149,22 @@ private: using DependenceVector = SmallVector<DepInfo, 8>; SmallVector<DependenceVector *, 16> DependenceStack; - /// If not null, a set limiting the attribute opportunities. - const DenseSet<const char *> *Allowed; - - /// Whether to delete functions. - const bool DeleteFns; - - /// Whether to rewrite signatures. - const bool RewriteSignatures; - - /// Maximum number of fixedpoint iterations. - Optional<unsigned> MaxFixpointIterations; - /// A set to remember the functions we already assume to be live and visited. DenseSet<const Function *> VisitedFunctions; /// Uses we replace with a new value after manifest is done. We will remove /// then trivially dead instructions as well. - DenseMap<Use *, Value *> ToBeChangedUses; + SmallMapVector<Use *, Value *, 32> ToBeChangedUses; /// Values we replace with a new value after manifest is done. We will remove /// then trivially dead instructions as well. - DenseMap<Value *, std::pair<Value *, bool>> ToBeChangedValues; + SmallMapVector<Value *, std::pair<Value *, bool>, 32> ToBeChangedValues; /// Instructions we replace with `unreachable` insts after manifest is done. - SmallDenseSet<WeakVH, 16> ToBeChangedToUnreachableInsts; + SmallSetVector<WeakVH, 16> ToBeChangedToUnreachableInsts; /// Invoke instructions with at least a single dead successor block. - SmallVector<WeakVH, 16> InvokeWithDeadSuccessor; + SmallSetVector<WeakVH, 16> InvokeWithDeadSuccessor; /// A flag that indicates which stage of the process we are in. Initially, the /// phase is SEEDING. Phase is changed in `Attributor::run()` @@ -2083,21 +2181,18 @@ private: /// Functions, blocks, and instructions we delete after manifest is done. /// ///{ - SmallPtrSet<Function *, 8> ToBeDeletedFunctions; - SmallPtrSet<BasicBlock *, 8> ToBeDeletedBlocks; SmallPtrSet<BasicBlock *, 8> ManifestAddedBlocks; - SmallDenseSet<WeakVH, 8> ToBeDeletedInsts; + SmallSetVector<Function *, 8> ToBeDeletedFunctions; + SmallSetVector<BasicBlock *, 8> ToBeDeletedBlocks; + SmallSetVector<WeakVH, 8> ToBeDeletedInsts; ///} - /// Callback to get an OptimizationRemarkEmitter from a Function *. - Optional<OptimizationRemarkGetter> OREGetter; - /// Container with all the query AAs that requested an update via /// registerForUpdate. SmallSetVector<AbstractAttribute *, 16> QueryAAsAwaitingUpdate; - /// The name of the pass to emit remarks for. - const char *PassName = ""; + /// User provided configuration for this Attributor instance. + const AttributorConfig Configuration; friend AADepGraph; friend AttributorCallGraph; @@ -2515,16 +2610,6 @@ struct IntegerRangeState : public AbstractState { unionAssumed(R.getAssumed()); } - /// Unite known range with the passed state. - void unionKnown(const ConstantRange &R) { - // Don't loose a known range. - Known = Known.unionWith(R); - Assumed = Assumed.unionWith(Known); - } - - /// See IntegerRangeState::unionKnown(..). - void unionKnown(const IntegerRangeState &R) { unionKnown(R.getKnown()); } - /// Intersect known range with the passed state. void intersectKnown(const ConstantRange &R) { Assumed = Assumed.intersectWith(R); @@ -2554,8 +2639,8 @@ struct IntegerRangeState : public AbstractState { IntegerRangeState operator&=(const IntegerRangeState &R) { // NOTE: `&=` operator seems like `intersect` but in this case, we need to // take `union`. - unionKnown(R); - unionAssumed(R); + Known = Known.unionWith(R.getKnown()); + Assumed = Assumed.unionWith(R.getAssumed()); return *this; } }; @@ -3363,6 +3448,12 @@ protected: /// Returns true if \p I is known dead. virtual bool isKnownDead(const Instruction *I) const = 0; + /// Return true if the underlying value is a store that is known to be + /// removable. This is different from dead stores as the removable store + /// can have an effect on live values, especially loads, but that effect + /// is propagated which allows us to remove the store in turn. + virtual bool isRemovableStore() const { return false; } + /// This method is used to check if at least one instruction in a collection /// of instructions is live. template <typename T> bool isLiveInstSet(T begin, T end) const { @@ -3618,10 +3709,10 @@ struct AAAlign : public IRAttribute< AAAlign(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {} /// Return assumed alignment. - uint64_t getAssumedAlign() const { return getAssumed(); } + Align getAssumedAlign() const { return Align(getAssumed()); } /// Return known alignment. - uint64_t getKnownAlign() const { return getKnown(); } + Align getKnownAlign() const { return Align(getKnown()); } /// See AbstractAttribute::getName() const std::string getName() const override { return "AAAlign"; } @@ -3641,6 +3732,46 @@ struct AAAlign : public IRAttribute< static const char ID; }; +/// An abstract interface to track if a value leaves it's defining function +/// instance. +/// TODO: We should make it a ternary AA tracking uniqueness, and uniqueness +/// wrt. the Attributor analysis separately. +struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> { + AAInstanceInfo(const IRPosition &IRP, Attributor &A) + : StateWrapper<BooleanState, AbstractAttribute>(IRP) {} + + /// Return true if we know that the underlying value is unique in its scope + /// wrt. the Attributor analysis. That means it might not be unique but we can + /// still use pointer equality without risking to represent two instances with + /// one `llvm::Value`. + bool isKnownUniqueForAnalysis() const { return isKnown(); } + + /// Return true if we assume that the underlying value is unique in its scope + /// wrt. the Attributor analysis. That means it might not be unique but we can + /// still use pointer equality without risking to represent two instances with + /// one `llvm::Value`. + bool isAssumedUniqueForAnalysis() const { return isAssumed(); } + + /// Create an abstract attribute view for the position \p IRP. + static AAInstanceInfo &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// See AbstractAttribute::getName() + const std::string getName() const override { return "AAInstanceInfo"; } + + /// See AbstractAttribute::getIdAddr() + const char *getIdAddr() const override { return &ID; } + + /// This function should return true if the type of the \p AA is + /// AAInstanceInfo + static bool classof(const AbstractAttribute *AA) { + return (AA->getIdAddr() == &ID); + } + + /// Unique ID (due to the unique address) + static const char ID; +}; + /// An abstract interface for all nocapture attributes. struct AANoCapture : public IRAttribute< @@ -4150,13 +4281,14 @@ struct AAValueConstantRange /// Return an assumed constant for the associated value a program point \p /// CtxI. - Optional<ConstantInt *> - getAssumedConstantInt(Attributor &A, - const Instruction *CtxI = nullptr) const { + Optional<Constant *> + getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const { ConstantRange RangeV = getAssumedConstantRange(A, CtxI); - if (auto *C = RangeV.getSingleElement()) - return cast<ConstantInt>( - ConstantInt::get(getAssociatedValue().getType(), *C)); + if (auto *C = RangeV.getSingleElement()) { + Type *Ty = getAssociatedValue().getType(); + return cast_or_null<Constant>( + AA::getWithType(*ConstantInt::get(Ty->getContext(), *C), *Ty)); + } if (RangeV.isEmptySet()) return llvm::None; return nullptr; @@ -4185,10 +4317,9 @@ struct AAValueConstantRange /// contains every possible value (i.e. we cannot in any way limit the value /// that the target position can take). That never happens naturally, we only /// force it. As for the conditions under which we force it, see -/// AAPotentialValues. -template <typename MemberTy, typename KeyInfo = DenseMapInfo<MemberTy>> -struct PotentialValuesState : AbstractState { - using SetTy = DenseSet<MemberTy, KeyInfo>; +/// AAPotentialConstantValues. +template <typename MemberTy> struct PotentialValuesState : AbstractState { + using SetTy = SmallSetVector<MemberTy, 8>; PotentialValuesState() : IsValidState(true), UndefIsContained(false) {} @@ -4247,7 +4378,7 @@ struct PotentialValuesState : AbstractState { return PotentialValuesState(true); } - static PotentialValuesState getBestState(PotentialValuesState &PVS) { + static PotentialValuesState getBestState(const PotentialValuesState &PVS) { return getBestState(); } @@ -4278,6 +4409,12 @@ struct PotentialValuesState : AbstractState { return *this; } +protected: + SetTy &getAssumedSet() { + assert(isValidState() && "This set shoud not be used when it is invalid!"); + return Set; + } + private: /// Check the size of this set, and invalidate when the size is no /// less than \p MaxPotentialValues threshold. @@ -4372,10 +4509,10 @@ raw_ostream &operator<<(raw_ostream &OS, /// operator we do not currently handle). /// /// TODO: Support values other than constant integers. -struct AAPotentialValues +struct AAPotentialConstantValues : public StateWrapper<PotentialConstantIntValuesState, AbstractAttribute> { using Base = StateWrapper<PotentialConstantIntValuesState, AbstractAttribute>; - AAPotentialValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {} + AAPotentialConstantValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {} /// See AbstractAttribute::getState(...). PotentialConstantIntValuesState &getState() override { return *this; } @@ -4384,22 +4521,23 @@ struct AAPotentialValues } /// Create an abstract attribute view for the position \p IRP. - static AAPotentialValues &createForPosition(const IRPosition &IRP, - Attributor &A); + static AAPotentialConstantValues &createForPosition(const IRPosition &IRP, + Attributor &A); /// Return assumed constant for the associated value - Optional<ConstantInt *> - getAssumedConstantInt(Attributor &A, - const Instruction *CtxI = nullptr) const { + Optional<Constant *> + getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const { if (!isValidState()) return nullptr; - if (getAssumedSet().size() == 1) - return cast<ConstantInt>(ConstantInt::get(getAssociatedValue().getType(), - *(getAssumedSet().begin()))); + if (getAssumedSet().size() == 1) { + Type *Ty = getAssociatedValue().getType(); + return cast_or_null<Constant>(AA::getWithType( + *ConstantInt::get(Ty->getContext(), *(getAssumedSet().begin())), + *Ty)); + } if (getAssumedSet().size() == 0) { if (undefIsContained()) - return cast<ConstantInt>( - ConstantInt::get(getAssociatedValue().getType(), 0)); + return UndefValue::get(getAssociatedValue().getType()); return llvm::None; } @@ -4407,13 +4545,15 @@ struct AAPotentialValues } /// See AbstractAttribute::getName() - const std::string getName() const override { return "AAPotentialValues"; } + const std::string getName() const override { + return "AAPotentialConstantValues"; + } /// See AbstractAttribute::getIdAddr() const char *getIdAddr() const override { return &ID; } /// This function should return true if the type of the \p AA is - /// AAPotentialValues + /// AAPotentialConstantValues static bool classof(const AbstractAttribute *AA) { return (AA->getIdAddr() == &ID); } @@ -4744,12 +4884,10 @@ struct AAPointerInfo : public AbstractAttribute { Instruction *getRemoteInst() const { return RemoteI; } /// Return true if the value written is not known yet. - bool isWrittenValueYetUndetermined() const { return !Content.hasValue(); } + bool isWrittenValueYetUndetermined() const { return !Content; } /// Return true if the value written cannot be determined at all. - bool isWrittenValueUnknown() const { - return Content.hasValue() && !*Content; - } + bool isWrittenValueUnknown() const { return Content && !*Content; } /// Return the type associated with the access, if known. Type *getType() const { return Ty; } @@ -4792,21 +4930,55 @@ struct AAPointerInfo : public AbstractAttribute { /// See AbstractAttribute::getIdAddr() const char *getIdAddr() const override { return &ID; } - /// Call \p CB on all accesses that might interfere with \p LI and return true - /// if all such accesses were known and the callback returned true for all of - /// them, false otherwise. - virtual bool forallInterferingAccesses( - LoadInst &LI, function_ref<bool(const Access &, bool)> CB) const = 0; + /// Helper to represent an access offset and size, with logic to deal with + /// uncertainty and check for overlapping accesses. + struct OffsetAndSize : public std::pair<int64_t, int64_t> { + using BaseTy = std::pair<int64_t, int64_t>; + OffsetAndSize(int64_t Offset, int64_t Size) : BaseTy(Offset, Size) {} + OffsetAndSize(const BaseTy &P) : BaseTy(P) {} + int64_t getOffset() const { return first; } + int64_t getSize() const { return second; } + static OffsetAndSize getUnknown() { + return OffsetAndSize(Unknown, Unknown); + } + + /// Return true if offset or size are unknown. + bool offsetOrSizeAreUnknown() const { + return getOffset() == OffsetAndSize::Unknown || + getSize() == OffsetAndSize::Unknown; + } + + /// Return true if this offset and size pair might describe an address that + /// overlaps with \p OAS. + bool mayOverlap(const OffsetAndSize &OAS) const { + // Any unknown value and we are giving up -> overlap. + if (offsetOrSizeAreUnknown() || OAS.offsetOrSizeAreUnknown()) + return true; + + // Check if one offset point is in the other interval [offset, + // offset+size]. + return OAS.getOffset() + OAS.getSize() > getOffset() && + OAS.getOffset() < getOffset() + getSize(); + } + + /// Constant used to represent unknown offset or sizes. + static constexpr int64_t Unknown = 1 << 31; + }; + + /// Call \p CB on all accesses that might interfere with \p OAS and return + /// true if all such accesses were known and the callback returned true for + /// all of them, false otherwise. An access interferes with an offset-size + /// pair if it might read or write that memory region. virtual bool forallInterferingAccesses( - StoreInst &SI, function_ref<bool(const Access &, bool)> CB) const = 0; + OffsetAndSize OAS, function_ref<bool(const Access &, bool)> CB) const = 0; - /// Call \p CB on all write accesses that might interfere with \p LI and + /// Call \p CB on all accesses that might interfere with \p I and /// return true if all such accesses were known and the callback returned true /// for all of them, false otherwise. In contrast to forallInterferingAccesses /// this function will perform reasoning to exclude write accesses that cannot /// affect the load even if they on the surface look as if they would. - virtual bool forallInterferingWrites( - Attributor &A, const AbstractAttribute &QueryingAA, LoadInst &LI, + virtual bool forallInterferingAccesses( + Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I, function_ref<bool(const Access &, bool)> CB) const = 0; /// This function should return true if the type of the \p AA is AAPointerInfo diff --git a/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h b/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h index 496ceea12bc9..a71fa3bf404d 100644 --- a/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h +++ b/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h @@ -66,25 +66,24 @@ public: } }; - /// Liveness enum - During our initial pass over the program, we determine - /// that things are either alive or maybe alive. We don't mark anything - /// explicitly dead (even if we know they are), since anything not alive - /// with no registered uses (in Uses) will never be marked alive and will - /// thus become dead in the end. + /// During our initial pass over the program, we determine that things are + /// either alive or maybe alive. We don't mark anything explicitly dead (even + /// if we know they are), since anything not alive with no registered uses + /// (in Uses) will never be marked alive and will thus become dead in the end. enum Liveness { Live, MaybeLive }; - DeadArgumentEliminationPass(bool ShouldHackArguments_ = false) - : ShouldHackArguments(ShouldHackArguments_) {} + DeadArgumentEliminationPass(bool ShouldHackArguments = false) + : ShouldHackArguments(ShouldHackArguments) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &); /// Convenience wrapper - RetOrArg CreateRet(const Function *F, unsigned Idx) { + RetOrArg createRet(const Function *F, unsigned Idx) { return RetOrArg(F, Idx, false); } /// Convenience wrapper - RetOrArg CreateArg(const Function *F, unsigned Idx) { + RetOrArg createArg(const Function *F, unsigned Idx) { return RetOrArg(F, Idx, true); } @@ -122,21 +121,21 @@ public: bool ShouldHackArguments = false; private: - Liveness MarkIfNotLive(RetOrArg Use, UseVector &MaybeLiveUses); - Liveness SurveyUse(const Use *U, UseVector &MaybeLiveUses, + Liveness markIfNotLive(RetOrArg Use, UseVector &MaybeLiveUses); + Liveness surveyUse(const Use *U, UseVector &MaybeLiveUses, unsigned RetValNum = -1U); - Liveness SurveyUses(const Value *V, UseVector &MaybeLiveUses); + Liveness surveyUses(const Value *V, UseVector &MaybeLiveUses); - void SurveyFunction(const Function &F); - bool IsLive(const RetOrArg &RA); - void MarkValue(const RetOrArg &RA, Liveness L, + void surveyFunction(const Function &F); + bool isLive(const RetOrArg &RA); + void markValue(const RetOrArg &RA, Liveness L, const UseVector &MaybeLiveUses); - void MarkLive(const RetOrArg &RA); - void MarkLive(const Function &F); - void PropagateLiveness(const RetOrArg &RA); - bool RemoveDeadStuffFromFunction(Function *F); - bool DeleteDeadVarargs(Function &Fn); - bool RemoveDeadArgumentsFromCallers(Function &Fn); + void markLive(const RetOrArg &RA); + void markLive(const Function &F); + void propagateLiveness(const RetOrArg &RA); + bool removeDeadStuffFromFunction(Function *F); + bool deleteDeadVarargs(Function &F); + bool removeDeadArgumentsFromCallers(Function &F); }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/IPO/ForceFunctionAttrs.h b/llvm/include/llvm/Transforms/IPO/ForceFunctionAttrs.h index a2b93f8aa30d..07c7cac77354 100644 --- a/llvm/include/llvm/Transforms/IPO/ForceFunctionAttrs.h +++ b/llvm/include/llvm/Transforms/IPO/ForceFunctionAttrs.h @@ -14,9 +14,10 @@ #define LLVM_TRANSFORMS_IPO_FORCEFUNCTIONATTRS_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Module; +class Pass; /// Pass which forces specific function attributes into the IR, primarily as /// a debugging tool. diff --git a/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h b/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h index 0b6734a3929d..bcb75025f8e5 100644 --- a/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h +++ b/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h @@ -15,29 +15,22 @@ #ifndef LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H #define LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/LazyCallGraph.h" -#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/PassManager.h" namespace llvm { -class AAResults; +class GlobalValueSummary; +class ModuleSummaryIndex; class Function; class Module; class Pass; -/// The three kinds of memory access relevant to 'readonly' and -/// 'readnone' attributes. -enum MemoryAccessKind { - MAK_ReadNone = 0, - MAK_ReadOnly = 1, - MAK_MayWrite = 2, - MAK_WriteOnly = 3 -}; - /// Returns the memory access properties of this copy of the function. -MemoryAccessKind computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR); +FunctionModRefBehavior computeFunctionBodyMemoryAccess(Function &F, + AAResults &AAR); /// Propagate function attributes for function summaries along the index's /// callgraph during thinlink diff --git a/llvm/include/llvm/Transforms/IPO/GlobalDCE.h b/llvm/include/llvm/Transforms/IPO/GlobalDCE.h index 0a6851849e7e..a24196efb83b 100644 --- a/llvm/include/llvm/Transforms/IPO/GlobalDCE.h +++ b/llvm/include/llvm/Transforms/IPO/GlobalDCE.h @@ -19,11 +19,18 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/IR/Module.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" #include <unordered_map> namespace llvm { +class Comdat; +class Constant; +class Function; +class GlobalVariable; +class Metadata; +class Module; +class Value; /// Pass to remove unused function declarations. class GlobalDCEPass : public PassInfoMixin<GlobalDCEPass> { diff --git a/llvm/include/llvm/Transforms/IPO/IROutliner.h b/llvm/include/llvm/Transforms/IPO/IROutliner.h index e4807a1c9c65..315587e0f922 100644 --- a/llvm/include/llvm/Transforms/IPO/IROutliner.h +++ b/llvm/include/llvm/Transforms/IPO/IROutliner.h @@ -43,14 +43,13 @@ #include "llvm/Analysis/IRSimilarityIdentifier.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/ValueMap.h" #include "llvm/Support/InstructionCost.h" #include "llvm/Transforms/Utils/CodeExtractor.h" -#include <set> struct OutlinableGroup; namespace llvm { +using namespace CallingConv; using namespace IRSimilarity; class Module; @@ -86,6 +85,13 @@ struct OutlinableRegion { DenseMap<unsigned, unsigned> ExtractedArgToAgg; DenseMap<unsigned, unsigned> AggArgToExtracted; + /// Values in the outlined functions will often be replaced by arguments. When + /// finding corresponding values from one region to another, the found value + /// will be the value the argument previously replaced. This structure maps + /// any replaced values for the region to the aggregate aggregate argument + /// in the overall function. + DenseMap<Value *, Value *> RemappedArguments; + /// Marks whether we need to change the order of the arguments when mapping /// the old extracted function call to the new aggregate outlined function /// call. @@ -168,6 +174,15 @@ struct OutlinableRegion { /// \return The corresponding Value to \p V if it exists, otherwise nullptr. Value *findCorrespondingValueIn(const OutlinableRegion &Other, Value *V); + /// Find a corresponding BasicBlock for \p BB in similar OutlinableRegion \p Other. + /// + /// \param Other [in] - The OutlinableRegion to find the corresponding + /// BasicBlock in. + /// \param BB [in] - The BasicBlock to look for in the other region. + /// \return The corresponding Value to \p V if it exists, otherwise nullptr. + BasicBlock *findCorrespondingBlockIn(const OutlinableRegion &Other, + BasicBlock *BB); + /// Get the size of the code removed from the region. /// /// \param [in] TTI - The TargetTransformInfo for the parent function. @@ -372,6 +387,25 @@ private: // the call in outlined functions. if (CI.canReturnTwice()) return false; + // TODO: Update the outliner to capture whether the outlined function + // needs these extra attributes. + + // Functions marked with the swifttailcc and tailcc calling conventions + // require special handling when outlining musttail functions. The + // calling convention must be passed down to the outlined function as + // well. Further, there is special handling for musttail calls as well, + // requiring a return call directly after. For now, the outliner does not + // support this. + bool IsTailCC = CI.getCallingConv() == CallingConv::SwiftTail || + CI.getCallingConv() == CallingConv::Tail; + if (IsTailCC && !EnableMustTailCalls) + return false; + if (CI.isMustTailCall() && !EnableMustTailCalls) + return false; + // The outliner can only handle musttail items if it is also accompanied + // by the tailcc or swifttailcc calling convention. + if (CI.isMustTailCall() && !IsTailCC) + return false; return true; } // TODO: Handle FreezeInsts. Since a frozen value could be frozen inside @@ -397,6 +431,9 @@ private: // The flag variable that marks whether we should allow intrinsics // instructions to be outlined. bool EnableIntrinsics = false; + + // The flag variable that marks whether we should allow musttail calls. + bool EnableMustTailCalls = false; }; /// A InstVisitor used to exclude certain instructions from being outlined. diff --git a/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h b/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h index 302695d96355..880af2b46d7f 100644 --- a/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h +++ b/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h @@ -15,11 +15,11 @@ #ifndef LLVM_TRANSFORMS_IPO_INFERFUNCTIONATTRS_H #define LLVM_TRANSFORMS_IPO_INFERFUNCTIONATTRS_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Module; +class Pass; /// A pass which infers function attributes from the names and signatures of /// function declarations in a module. diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h index a7060943c4c0..1e154eb8f5da 100644 --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -16,7 +16,6 @@ #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/IR/PassManager.h" -#include <utility> namespace llvm { @@ -96,7 +95,9 @@ protected: /// passes be composed to achieve the same end result. class InlinerPass : public PassInfoMixin<InlinerPass> { public: - InlinerPass(bool OnlyMandatory = false) : OnlyMandatory(OnlyMandatory) {} + InlinerPass(bool OnlyMandatory = false, + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) + : OnlyMandatory(OnlyMandatory), LTOPhase(LTOPhase) {} InlinerPass(InlinerPass &&Arg) = default; PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, @@ -110,6 +111,7 @@ private: FunctionAnalysisManager &FAM, Module &M); std::unique_ptr<InlineAdvisor> OwnedAdvisor; const bool OnlyMandatory; + const ThinOrFullLTOPhase LTOPhase; }; /// Module pass, wrapping the inliner pass. This works in conjunction with the @@ -122,6 +124,7 @@ class ModuleInlinerWrapperPass public: ModuleInlinerWrapperPass( InlineParams Params = getInlineParams(), bool MandatoryFirst = true, + InlineContext IC = {}, InliningAdvisorMode Mode = InliningAdvisorMode::Default, unsigned MaxDevirtIterations = 0); ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default; @@ -147,6 +150,7 @@ public: private: const InlineParams Params; + const InlineContext IC; const InliningAdvisorMode Mode; const unsigned MaxDevirtIterations; // TODO: Clean this up so we only have one ModulePassManager. diff --git a/llvm/include/llvm/Transforms/IPO/Internalize.h b/llvm/include/llvm/Transforms/IPO/Internalize.h index 41816df93360..adcf5a932be0 100644 --- a/llvm/include/llvm/Transforms/IPO/Internalize.h +++ b/llvm/include/llvm/Transforms/IPO/Internalize.h @@ -23,7 +23,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringSet.h" -#include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" #include <functional> diff --git a/llvm/include/llvm/Transforms/IPO/ModuleInliner.h b/llvm/include/llvm/Transforms/IPO/ModuleInliner.h index 7474e48aafaf..24cfff6083ff 100644 --- a/llvm/include/llvm/Transforms/IPO/ModuleInliner.h +++ b/llvm/include/llvm/Transforms/IPO/ModuleInliner.h @@ -11,10 +11,7 @@ #include "llvm/Analysis/InlineAdvisor.h" #include "llvm/Analysis/InlineCost.h" -#include "llvm/Analysis/ReplayInlineAdvisor.h" -#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/IR/PassManager.h" -#include <utility> namespace llvm { @@ -30,8 +27,9 @@ namespace llvm { class ModuleInlinerPass : public PassInfoMixin<ModuleInlinerPass> { public: ModuleInlinerPass(InlineParams Params = getInlineParams(), - InliningAdvisorMode Mode = InliningAdvisorMode::Default) - : Params(Params), Mode(Mode){}; + InliningAdvisorMode Mode = InliningAdvisorMode::Default, + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) + : Params(Params), Mode(Mode), LTOPhase(LTOPhase){}; ModuleInlinerPass(ModuleInlinerPass &&Arg) = default; PreservedAnalyses run(Module &, ModuleAnalysisManager &); @@ -42,6 +40,7 @@ private: std::unique_ptr<InlineAdvisor> OwnedAdvisor; const InlineParams Params; const InliningAdvisorMode Mode; + const ThinOrFullLTOPhase LTOPhase; }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h index 3b944878a810..2676f2705424 100644 --- a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h +++ b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -16,7 +16,6 @@ #include "llvm-c/Transforms/PassManagerBuilder.h" #include <functional> -#include <memory> #include <string> #include <vector> @@ -214,7 +213,6 @@ private: void addInitialAliasAnalysisPasses(legacy::PassManagerBase &PM) const; void addLTOOptimizationPasses(legacy::PassManagerBase &PM); void addLateLTOOptimizationPasses(legacy::PassManagerBase &PM); - void addPGOInstrPasses(legacy::PassManagerBase &MPM, bool IsCS); void addFunctionSimplificationPasses(legacy::PassManagerBase &MPM); void addVectorPasses(legacy::PassManagerBase &PM, bool IsFullLTO); @@ -226,8 +224,6 @@ public: /// populateModulePassManager - This sets up the primary pass manager. void populateModulePassManager(legacy::PassManagerBase &MPM); - void populateLTOPassManager(legacy::PassManagerBase &PM); - void populateThinLTOPassManager(legacy::PassManagerBase &PM); }; /// Registers a function for adding a standard set of passes. This should be diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h index 893654650caa..fff06da22cf3 100644 --- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h +++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h @@ -18,9 +18,6 @@ #include <queue> #include <set> -using namespace llvm; -using namespace sampleprof; - namespace llvm { namespace sampleprof { @@ -51,10 +48,10 @@ struct ProfiledCallGraphNode { } }; - using iterator = std::set<ProfiledCallGraphEdge>::iterator; - using const_iterator = std::set<ProfiledCallGraphEdge>::const_iterator; using edge = ProfiledCallGraphEdge; - using edges = std::set<ProfiledCallGraphEdge, ProfiledCallGraphEdgeComparer>; + using edges = std::set<edge, ProfiledCallGraphEdgeComparer>; + using iterator = edges::iterator; + using const_iterator = edges::const_iterator; ProfiledCallGraphNode(StringRef FName = StringRef()) : Name(FName) {} @@ -64,11 +61,11 @@ struct ProfiledCallGraphNode { class ProfiledCallGraph { public: - using iterator = std::set<ProfiledCallGraphEdge>::iterator; + using iterator = ProfiledCallGraphNode::iterator; // Constructor for non-CS profile. ProfiledCallGraph(SampleProfileMap &ProfileMap) { - assert(!FunctionSamples::ProfileIsCSFlat && + assert(!FunctionSamples::ProfileIsCS && "CS flat profile is not handled here"); for (const auto &Samples : ProfileMap) { addProfiledCalls(Samples.second); diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h index cf87d028600f..a97d5ee3d710 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -15,20 +15,18 @@ #ifndef LLVM_TRANSFORMS_IPO_SAMPLECONTEXTTRACKER_H #define LLVM_TRANSFORMS_IPO_SAMPLECONTEXTTRACKER_H -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/Instructions.h" #include "llvm/ProfileData/SampleProf.h" -#include <list> #include <map> +#include <queue> #include <vector> -using namespace llvm; -using namespace sampleprof; - namespace llvm { +class CallBase; +class DILocation; +class Function; +class Instruction; // Internal trie tree representation used for tracking context tree and sample // profiles. The path from root node to a given node represents the context of @@ -47,11 +45,6 @@ public: ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite, StringRef ChildName, bool AllowCreate = true); - - ContextTrieNode &moveToChildContext(const LineLocation &CallSite, - ContextTrieNode &&NodeToMove, - uint32_t ContextFramesToRemove, - bool DeleteNode = true); void removeChildContext(const LineLocation &CallSite, StringRef ChildName); std::map<uint64_t, ContextTrieNode> &getAllChildContext(); StringRef getFuncName() const; @@ -62,6 +55,7 @@ public: LineLocation getCallSiteLoc() const; ContextTrieNode *getParentContext() const; void setParentContext(ContextTrieNode *Parent); + void setCallSiteLoc(const LineLocation &Loc); void dumpNode(); void dumpTree(); @@ -94,22 +88,13 @@ private: // calling context and the context is identified by path from root to the node. class SampleContextTracker { public: - struct ProfileComparer { - bool operator()(FunctionSamples *A, FunctionSamples *B) const { - // Sort function profiles by the number of total samples and their - // contexts. - if (A->getTotalSamples() == B->getTotalSamples()) - return A->getContext() < B->getContext(); - return A->getTotalSamples() > B->getTotalSamples(); - } - }; - - // Keep profiles of a function sorted so that they will be processed/promoted - // deterministically. - using ContextSamplesTy = std::set<FunctionSamples *, ProfileComparer>; + using ContextSamplesTy = std::vector<FunctionSamples *>; + SampleContextTracker() = default; SampleContextTracker(SampleProfileMap &Profiles, const DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap); + // Populate the FuncToCtxtProfiles map after the trie is built. + void populateFuncToCtxtMap(); // Query context profile for a specific callee with given name at a given // call-site. The full context is identified by location of call instruction. FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst, @@ -125,6 +110,8 @@ public: // Get all context profile for given function. ContextSamplesTy &getAllContextSamplesFor(const Function &Func); ContextSamplesTy &getAllContextSamplesFor(StringRef Name); + ContextTrieNode *getOrCreateContextPath(const SampleContext &Context, + bool AllowCreate); // Query base profile for a given function. A base profile is a merged view // of all context profiles for contexts that are not inlined. FunctionSamples *getBaseSamplesFor(const Function &Func, @@ -142,6 +129,64 @@ public: ContextTrieNode &getRootContext(); void promoteMergeContextSamplesTree(const Instruction &Inst, StringRef CalleeName); + + // Create a merged conext-less profile map. + void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles); + ContextTrieNode * + getContextNodeForProfile(const FunctionSamples *FSamples) const { + auto I = ProfileToNodeMap.find(FSamples); + if (I == ProfileToNodeMap.end()) + return nullptr; + return I->second; + } + StringMap<ContextSamplesTy> &getFuncToCtxtProfiles() { + return FuncToCtxtProfiles; + } + + class Iterator : public std::iterator<std::forward_iterator_tag, + const ContextTrieNode *> { + std::queue<ContextTrieNode *> NodeQueue; + + public: + explicit Iterator() = default; + explicit Iterator(ContextTrieNode *Node) { NodeQueue.push(Node); } + Iterator &operator++() { + assert(!NodeQueue.empty() && "Iterator already at the end"); + ContextTrieNode *Node = NodeQueue.front(); + NodeQueue.pop(); + for (auto &It : Node->getAllChildContext()) + NodeQueue.push(&It.second); + return *this; + } + + Iterator operator++(int) { + assert(!NodeQueue.empty() && "Iterator already at the end"); + Iterator Ret = *this; + ++(*this); + return Ret; + } + bool operator==(const Iterator &Other) const { + if (NodeQueue.empty() && Other.NodeQueue.empty()) + return true; + if (NodeQueue.empty() || Other.NodeQueue.empty()) + return false; + return NodeQueue.front() == Other.NodeQueue.front(); + } + bool operator!=(const Iterator &Other) const { return !(*this == Other); } + ContextTrieNode *operator*() const { + assert(!NodeQueue.empty() && "Invalid access to end iterator"); + return NodeQueue.front(); + } + }; + + Iterator begin() { return Iterator(&RootContext); } + Iterator end() { return Iterator(); } + +#ifndef NDEBUG + // Get a context string from root to current node. + std::string getContextString(const FunctionSamples &FSamples) const; + std::string getContextString(ContextTrieNode *Node) const; +#endif // Dump the internal context profile trie. void dump(); @@ -149,21 +194,26 @@ private: ContextTrieNode *getContextFor(const DILocation *DIL); ContextTrieNode *getCalleeContextFor(const DILocation *DIL, StringRef CalleeName); - ContextTrieNode *getOrCreateContextPath(const SampleContext &Context, - bool AllowCreate); ContextTrieNode *getTopLevelContextNode(StringRef FName); ContextTrieNode &addTopLevelContextNode(StringRef FName); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo); - void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode, - uint32_t ContextFramesToRemove); + void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode); ContextTrieNode & promoteMergeContextSamplesTree(ContextTrieNode &FromNode, - ContextTrieNode &ToNodeParent, - uint32_t ContextFramesToRemove); - + ContextTrieNode &ToNodeParent); + ContextTrieNode &moveContextSamples(ContextTrieNode &ToNodeParent, + const LineLocation &CallSite, + ContextTrieNode &&NodeToMove); + void setContextNode(const FunctionSamples *FSample, ContextTrieNode *Node) { + ProfileToNodeMap[FSample] = Node; + } // Map from function name to context profiles (excluding base profile) StringMap<ContextSamplesTy> FuncToCtxtProfiles; + // Map from current FunctionSample to the belonged context trie. + std::unordered_map<const FunctionSamples *, ContextTrieNode *> + ProfileToNodeMap; + // Map from function guid to real function names. Only used in md5 mode. const DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap; diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfile.h b/llvm/include/llvm/Transforms/IPO/SampleProfile.h index 704b793ab3ea..d838c8b8a83e 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleProfile.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfile.h @@ -36,7 +36,7 @@ public: private: std::string ProfileFileName; std::string ProfileRemappingFileName; - ThinOrFullLTOPhase LTOPhase; + const ThinOrFullLTOPhase LTOPhase; }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h index e73c36043cb2..ed296d2dd080 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -16,17 +16,19 @@ #define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H #include "llvm/ADT/DenseMap.h" -#include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/Analysis/LazyCallGraph.h" -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/PseudoProbe.h" #include "llvm/ProfileData/SampleProf.h" -#include "llvm/Target/TargetMachine.h" #include <unordered_map> namespace llvm { +class Any; +class BasicBlock; +class Function; +class Instruction; +class Loop; +class PassInstrumentationCallbacks; +class TargetMachine; class Module; diff --git a/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h b/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h index f4a15c36afc9..4a2eaad63113 100644 --- a/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h +++ b/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h @@ -16,11 +16,12 @@ #ifndef LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H #define LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// Pass to remove unused function declarations. struct StripDeadPrototypesPass : PassInfoMixin<StripDeadPrototypesPass> { PreservedAnalyses run(Module &M, ModuleAnalysisManager &); diff --git a/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h b/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h index 7acb922b37e1..469cf2bc5011 100644 --- a/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h +++ b/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h @@ -17,9 +17,10 @@ #define LLVM_TRANSFORMS_IPO_THINLTOBITCODEWRITER_H #include <llvm/IR/PassManager.h> -#include <llvm/Support/raw_ostream.h> namespace llvm { +class Module; +class raw_ostream; class ThinLTOBitcodeWriterPass : public PassInfoMixin<ThinLTOBitcodeWriterPass> { diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h index 2e9744cfd524..47c137e70a7f 100644 --- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -14,16 +14,17 @@ #ifndef LLVM_TRANSFORMS_IPO_WHOLEPROGRAMDEVIRT_H #define LLVM_TRANSFORMS_IPO_WHOLEPROGRAMDEVIRT_H -#include "llvm/IR/Module.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/IPO/FunctionImport.h" #include <cassert> #include <cstdint> +#include <map> #include <set> #include <utility> #include <vector> namespace llvm { +class Module; template <typename T> class ArrayRef; template <typename T> class MutableArrayRef; diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h index 6dee38c83b36..35a3a8c3218b 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h @@ -18,6 +18,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" #define DEBUG_TYPE "instcombine" #include "llvm/Transforms/Utils/InstructionWorklist.h" diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h index a288a3972c3d..9ff45fc29b06 100644 --- a/llvm/include/llvm/Transforms/Instrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation.h @@ -15,6 +15,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" #include <cassert> #include <cstdint> #include <limits> @@ -75,21 +79,6 @@ struct GCOVOptions { std::string Exclude; }; -ModulePass *createGCOVProfilerPass(const GCOVOptions &Options = - GCOVOptions::getDefault()); - -// PGO Instrumention. Parameter IsCS indicates if this is the context sensitive -// instrumentation. -ModulePass *createPGOInstrumentationGenLegacyPass(bool IsCS = false); -ModulePass * -createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""), - bool IsCS = false); -ModulePass *createPGOInstrumentationGenCreateVarLegacyPass( - StringRef CSInstrName = StringRef("")); -ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false, - bool SamplePGO = false); -FunctionPass *createPGOMemOPSizeOptLegacyPass(); - ModulePass *createCGProfileLegacyPass(); // The pgo-specific indirect call promotion function declared below is used by @@ -194,6 +183,26 @@ static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) { assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits"); return Scaled; } + +// Use to ensure the inserted instrumentation has a DebugLocation; if none is +// attached to the source instruction, try to use a DILocation with offset 0 +// scoped to surrounding function (if it has a DebugLocation). +// +// Some non-call instructions may be missing debug info, but when inserting +// instrumentation calls, some builds (e.g. LTO) want calls to have debug info +// if the enclosing function does. +struct InstrumentationIRBuilder : IRBuilder<> { + static void ensureDebugInfo(IRBuilder<> &IRB, const Function &F) { + if (IRB.getCurrentDebugLocation()) + return; + if (DISubprogram *SP = F.getSubprogram()) + IRB.SetCurrentDebugLocation(DILocation::get(SP->getContext(), 0, 0, SP)); + } + + explicit InstrumentationIRBuilder(Instruction *IP) : IRBuilder<>(IP) { + ensureDebugInfo(*this, *IP->getFunction()); + } +}; } // end namespace llvm #endif // LLVM_TRANSFORMS_INSTRUMENTATION_H diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h index a0d8118c23f7..d12b2cf45825 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h @@ -13,82 +13,17 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_ADDRESSSANITIZER_H #define LLVM_TRANSFORMS_INSTRUMENTATION_ADDRESSSANITIZER_H -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" #include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h" namespace llvm { - -/// Frontend-provided metadata for source location. -struct LocationMetadata { - StringRef Filename; - int LineNo = 0; - int ColumnNo = 0; - - LocationMetadata() = default; - - bool empty() const { return Filename.empty(); } - void parse(MDNode *MDN); -}; - -/// Frontend-provided metadata for global variables. -class GlobalsMetadata { -public: - struct Entry { - LocationMetadata SourceLoc; - StringRef Name; - bool IsDynInit = false; - bool IsExcluded = false; - - Entry() = default; - }; - - /// Create a default uninitialized GlobalsMetadata instance. - GlobalsMetadata() = default; - - /// Create an initialized GlobalsMetadata instance. - GlobalsMetadata(Module &M); - - /// Returns metadata entry for a given global. - Entry get(GlobalVariable *G) const { - auto Pos = Entries.find(G); - return (Pos != Entries.end()) ? Pos->second : Entry(); - } - - /// Handle invalidation from the pass manager. - /// These results are never invalidated. - bool invalidate(Module &, const PreservedAnalyses &, - ModuleAnalysisManager::Invalidator &) { - return false; - } - bool invalidate(Function &, const PreservedAnalyses &, - FunctionAnalysisManager::Invalidator &) { - return false; - } - -private: - DenseMap<GlobalVariable *, Entry> Entries; -}; - -/// The ASanGlobalsMetadataAnalysis initializes and returns a GlobalsMetadata -/// object. More specifically, ASan requires looking at all globals registered -/// in 'llvm.asan.globals' before running, which only depends on reading module -/// level metadata. This analysis is required to run before running the -/// AddressSanitizerPass since it collects that metadata. -/// The legacy pass manager equivalent of this is ASanGlobalsMetadataLegacyPass. -class ASanGlobalsMetadataAnalysis - : public AnalysisInfoMixin<ASanGlobalsMetadataAnalysis> { -public: - using Result = GlobalsMetadata; - - Result run(Module &, ModuleAnalysisManager &); - -private: - friend AnalysisInfoMixin<ASanGlobalsMetadataAnalysis>; - static AnalysisKey Key; -}; +class Function; +class FunctionPass; +class GlobalVariable; +class MDNode; +class Module; +class ModulePass; +class raw_ostream; struct AddressSanitizerOptions { bool CompileKernel = false; @@ -98,26 +33,6 @@ struct AddressSanitizerOptions { AsanDetectStackUseAfterReturnMode::Runtime; }; -/// Public interface to the address sanitizer pass for instrumenting code to -/// check for various memory errors at runtime. -/// -/// The sanitizer itself is a function pass that works by inserting various -/// calls to the ASan runtime library functions. The runtime library essentially -/// replaces malloc() and free() with custom implementations that allow regions -/// surrounding requested memory to be checked for invalid accesses. -class AddressSanitizerPass : public PassInfoMixin<AddressSanitizerPass> { -public: - AddressSanitizerPass(const AddressSanitizerOptions &Options) - : Options(Options){}; - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); - void printPipeline(raw_ostream &OS, - function_ref<StringRef(StringRef)> MapClassName2PassName); - static bool isRequired() { return true; } - -private: - AddressSanitizerOptions Options; -}; - /// Public interface to the address sanitizer module pass for instrumenting code /// to check for various memory errors. /// @@ -142,17 +57,6 @@ private: AsanDtorKind DestructorKind; }; -// Insert AddressSanitizer (address basic correctness checking) instrumentation -FunctionPass *createAddressSanitizerFunctionPass( - bool CompileKernel = false, bool Recover = false, - bool UseAfterScope = false, - AsanDetectStackUseAfterReturnMode UseAfterReturn = - AsanDetectStackUseAfterReturnMode::Runtime); -ModulePass *createModuleAddressSanitizerLegacyPassPass( - bool CompileKernel = false, bool Recover = false, bool UseGlobalsGC = true, - bool UseOdrIndicator = true, - AsanDtorKind DestructorKind = AsanDtorKind::Global); - struct ASanAccessInfo { const int32_t Packed; const uint8_t AccessSizeIndex; diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h index 0a5456c5956f..7858a1c4b2fd 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h @@ -47,51 +47,6 @@ public: Value *getPtr() { return PtrUse->get(); } }; -// For an alloca valid between lifetime markers Start and Ends, call the -// Callback for all possible exits out of the lifetime in the containing -// function, which can return from the instructions in RetVec. -// -// Returns whether Ends covered all possible exits. If they did not, -// the caller should remove Ends to ensure that work done at the other -// exits does not happen outside of the lifetime. -template <typename F> -bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, - const Instruction *Start, - const SmallVectorImpl<IntrinsicInst *> &Ends, - const SmallVectorImpl<Instruction *> &RetVec, - F Callback) { - if (Ends.size() == 1 && PDT.dominates(Ends[0], Start)) { - Callback(Ends[0]); - return true; - } - SmallVector<Instruction *, 8> ReachableRetVec; - unsigned NumCoveredExits = 0; - for (auto *RI : RetVec) { - if (!isPotentiallyReachable(Start, RI, nullptr, &DT)) - continue; - ReachableRetVec.push_back(RI); - // TODO(fmayer): We don't support diamond shapes, where multiple lifetime - // ends together dominate the RI, but none of them does by itself. - // Check how often this happens and decide whether to support this here. - if (std::any_of(Ends.begin(), Ends.end(), - [&](Instruction *End) { return DT.dominates(End, RI); })) - ++NumCoveredExits; - } - // If there's a mix of covered and non-covered exits, just put the untag - // on exits, so we avoid the redundancy of untagging twice. - if (NumCoveredExits == ReachableRetVec.size()) { - for (auto *End : Ends) - Callback(End); - } else { - for (auto *RI : ReachableRetVec) - Callback(RI); - // We may have inserted untag outside of the lifetime interval. - // Signal the caller to remove the lifetime end call for this alloca. - return false; - } - return true; -} - // Get AddressSanitizer parameters. void getAddressSanitizerParams(const Triple &TargetTriple, int LongSize, bool IsKasan, uint64_t *ShadowBase, diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h index f019d1c00a35..187aaedb6000 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h @@ -17,14 +17,13 @@ enum class AsanDtorKind { None, ///< Do not emit any destructors for ASan Global, ///< Append to llvm.global_dtors Invalid, ///< Not a valid destructor Kind. - // TODO(dliew): Add more more kinds. }; /// Mode of ASan detect stack use after return enum class AsanDetectStackUseAfterReturnMode { Never, ///< Never detect stack use after return. - Runtime, ///< Detect stack use after return if runtime flag is enabled - ///< (ASAN_OPTIONS=detect_stack_use_after_return=1) + Runtime, ///< Detect stack use after return if not disabled runtime with + ///< (ASAN_OPTIONS=detect_stack_use_after_return=0). Always, ///< Always detect stack use after return. Invalid, ///< Not a valid detect mode. }; diff --git a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h index 76d586252743..5e68141e3399 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h +++ b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h @@ -10,9 +10,10 @@ #define LLVM_TRANSFORMS_INSTRUMENTATION_BOUNDSCHECKING_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Function; +class FunctionPass; /// A pass to instrument code and perform run-time bounds checking on loads, /// stores, and other memory intrinsics. diff --git a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h index c56e4c78cad5..9f9ce42277a0 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h +++ b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h @@ -12,10 +12,10 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_CGPROFILE_H #define LLVM_TRANSFORMS_INSTRUMENTATION_CGPROFILE_H -#include "llvm/ADT/MapVector.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; class CGProfilePass : public PassInfoMixin<CGProfilePass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); diff --git a/llvm/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h b/llvm/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h index 18b428582046..0bace514c361 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h +++ b/llvm/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h @@ -14,7 +14,6 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_CONTROLHEIGHTREDUCTION_H #define LLVM_TRANSFORMS_INSTRUMENTATION_CONTROLHEIGHTREDUCTION_H -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h index 9b57b1f9a9ea..41ba05cd67f0 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h @@ -8,12 +8,12 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_DATAFLOWSANITIZER_H #define LLVM_TRANSFORMS_INSTRUMENTATION_DATAFLOWSANITIZER_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include <string> #include <vector> namespace llvm { +class Module; class DataFlowSanitizerPass : public PassInfoMixin<DataFlowSanitizerPass> { private: diff --git a/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h index 70949026a892..d3b5b5ca5c25 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h @@ -13,11 +13,14 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_HWADDRESSSANITIZER_H #define LLVM_TRANSFORMS_INSTRUMENTATION_HWADDRESSSANITIZER_H -#include "llvm/IR/Function.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class FunctionPass; +class Module; +class StringRef; +class raw_ostream; struct HWAddressSanitizerOptions { HWAddressSanitizerOptions() @@ -47,11 +50,6 @@ private: HWAddressSanitizerOptions Options; }; -FunctionPass * -createHWAddressSanitizerLegacyPassPass(bool CompileKernel = false, - bool Recover = false, - bool DisableOptimization = false); - namespace HWASanAccessInfo { // Bit field positions for the accessinfo parameter to diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index 5873db22a5d1..90fc0670448b 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -19,7 +19,6 @@ #include "llvm/IR/PassManager.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Transforms/Instrumentation.h" -#include <cstddef> #include <cstdint> #include <cstring> #include <vector> @@ -57,6 +56,9 @@ private: } }; DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap; + /// If runtime relocation is enabled, this maps functions to the load + /// instruction that produces the profile relocation bias. + DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap; std::vector<GlobalValue *> CompilerUsedVars; std::vector<GlobalValue *> UsedVars; std::vector<GlobalVariable *> ReferencedNames; diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h index b9ad56ba7509..b584b9984492 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h +++ b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h @@ -12,12 +12,13 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H #define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Function; +class FunctionPass; +class Module; +class ModulePass; /// Public interface to the memory profiler pass for instrumenting code to /// profile memory accesses. diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h index e5779dc775ba..e4654a0fc7ef 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h @@ -13,10 +13,15 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMORYSANITIZER_H #define LLVM_TRANSFORMS_INSTRUMENTATION_MEMORYSANITIZER_H +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Function; +class FunctionPass; +class Module; +class StringRef; +class raw_ostream; struct MemorySanitizerOptions { MemorySanitizerOptions() : MemorySanitizerOptions(0, false, false, false){}; @@ -30,10 +35,6 @@ struct MemorySanitizerOptions { bool EagerChecks; }; -// Insert MemorySanitizer instrumentation (detection of uninitialized reads) -FunctionPass * -createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options = {}); - /// A function pass for msan instrumentation. /// /// Instruments functions to detect unitialized reads. This function pass diff --git a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h index e3d268cb0781..9bacb7eb38a5 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h +++ b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h @@ -16,13 +16,14 @@ #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERCOVERAGE_H #define LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERCOVERAGE_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/SpecialCaseList.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Transforms/Instrumentation.h" namespace llvm { +class Module; +class ModulePass; /// This is the ModuleSanitizerCoverage pass used in the new pass manager. The /// pass instruments functions for coverage, adds initialization calls to the diff --git a/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h index e795043630d5..b3a067ba59c2 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h @@ -14,11 +14,11 @@ #define LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { -// Insert ThreadSanitizer (race detection) instrumentation -FunctionPass *createThreadSanitizerLegacyPassPass(); +class Function; +class FunctionPass; +class Module; /// A function pass for tsan instrumentation. /// diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h index d6228700aa9a..edd492b0343d 100644 --- a/llvm/include/llvm/Transforms/Scalar.h +++ b/llvm/include/llvm/Transforms/Scalar.h @@ -133,7 +133,8 @@ Pass *createIndVarSimplifyPass(); // Pass *createLICMPass(); Pass *createLICMPass(unsigned LicmMssaOptCap, - unsigned LicmMssaNoAccForPromotionCap); + unsigned LicmMssaNoAccForPromotionCap, + bool AllowSpeculation); //===----------------------------------------------------------------------===// // @@ -170,13 +171,6 @@ Pass *createLoopStrengthReducePass(); //===----------------------------------------------------------------------===// // -// LoopUnswitch - This pass is a simple loop unswitching pass. -// -Pass *createLoopUnswitchPass(bool OptimizeForSize = false, - bool hasBranchDivergence = false); - -//===----------------------------------------------------------------------===// -// // LoopInstSimplify - This pass simplifies instructions in a loop's body. // Pass *createLoopInstSimplifyPass(); @@ -246,12 +240,10 @@ FunctionPass *createReassociatePass(); //===----------------------------------------------------------------------===// // // JumpThreading - Thread control through mult-pred/multi-succ blocks where some -// preds always go to some succ. If FreezeSelectCond is true, unfold the -// condition of a select that unfolds to branch. Thresholds other than minus one +// preds always go to some succ. Thresholds other than minus one // override the internal BB duplication default threshold. // -FunctionPass *createJumpThreadingPass(bool FreezeSelectCond = false, - int Threshold = -1); +FunctionPass *createJumpThreadingPass(int Threshold = -1); //===----------------------------------------------------------------------===// // @@ -428,6 +420,12 @@ FunctionPass *createLowerExpectIntrinsicPass(); //===----------------------------------------------------------------------===// // +// TLSVariableHoist - This pass reduce duplicated TLS address call. +// +FunctionPass *createTLSVariableHoistPass(); + +//===----------------------------------------------------------------------===// +// // LowerConstantIntrinsicss - Expand any remaining llvm.objectsize and // llvm.is.constant intrinsic calls, even for the unknown cases. // diff --git a/llvm/include/llvm/Transforms/Scalar/BDCE.h b/llvm/include/llvm/Transforms/Scalar/BDCE.h index 996622bccdba..0763f31dfad4 100644 --- a/llvm/include/llvm/Transforms/Scalar/BDCE.h +++ b/llvm/include/llvm/Transforms/Scalar/BDCE.h @@ -16,11 +16,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_BDCE_H #define LLVM_TRANSFORMS_SCALAR_BDCE_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + // The Bit-Tracking Dead Code Elimination pass. struct BDCEPass : PassInfoMixin<BDCEPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); diff --git a/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h index ee2b6f264086..661340f4598f 100644 --- a/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h +++ b/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h @@ -9,11 +9,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING_H #define LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct CallSiteSplittingPass : PassInfoMixin<CallSiteSplittingPass> { /// Run the pass over the function. PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); diff --git a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h index 11379e59467f..e59734b92244 100644 --- a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h +++ b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h @@ -40,7 +40,6 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include <algorithm> diff --git a/llvm/include/llvm/Transforms/Scalar/DCE.h b/llvm/include/llvm/Transforms/Scalar/DCE.h index 4d83296b1d86..8d1616a7b75d 100644 --- a/llvm/include/llvm/Transforms/Scalar/DCE.h +++ b/llvm/include/llvm/Transforms/Scalar/DCE.h @@ -13,11 +13,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_DCE_H #define LLVM_TRANSFORMS_SCALAR_DCE_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// Basic Dead Code Elimination pass. class DCEPass : public PassInfoMixin<DCEPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/DFAJumpThreading.h b/llvm/include/llvm/Transforms/Scalar/DFAJumpThreading.h index afebd9bbc122..4e9fbf65e163 100644 --- a/llvm/include/llvm/Transforms/Scalar/DFAJumpThreading.h +++ b/llvm/include/llvm/Transforms/Scalar/DFAJumpThreading.h @@ -13,11 +13,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_DFAJUMPTHREADING_H #define LLVM_TRANSFORMS_SCALAR_DFAJUMPTHREADING_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct DFAJumpThreadingPass : PassInfoMixin<DFAJumpThreadingPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/Transforms/Scalar/Float2Int.h b/llvm/include/llvm/Transforms/Scalar/Float2Int.h index 5fb47af6f795..f4bec228ea96 100644 --- a/llvm/include/llvm/Transforms/Scalar/Float2Int.h +++ b/llvm/include/llvm/Transforms/Scalar/Float2Int.h @@ -18,11 +18,17 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SetVector.h" #include "llvm/IR/ConstantRange.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class DominatorTree; +class Function; +class Instruction; +class LLVMContext; +template <typename T> class Optional; +class Type; +class Value; + class Float2IntPass : public PassInfoMixin<Float2IntPass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -36,6 +42,7 @@ private: ConstantRange badRange(); ConstantRange unknownRange(); ConstantRange validateRange(ConstantRange R); + Optional<ConstantRange> calcRange(Instruction *I); void walkBackwards(); void walkForwards(); bool validateAndTransform(); diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index 9e660c92124e..16ab1a490162 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -17,10 +17,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" @@ -42,6 +40,8 @@ class CallInst; class ExtractValueInst; class Function; class FunctionPass; +class GetElementPtrInst; +class ImplicitControlFlowTracking; class LoadInst; class LoopInfo; class MemDepResult; @@ -178,6 +178,7 @@ public: Expression createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate, Value *LHS, Value *RHS); Expression createExtractvalueExpr(ExtractValueInst *EI); + Expression createGEPExpr(GetElementPtrInst *GEP); uint32_t lookupOrAddCall(CallInst *C); uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock, uint32_t Num, GVNPass &Gvn); diff --git a/llvm/include/llvm/Transforms/Scalar/GuardWidening.h b/llvm/include/llvm/Transforms/Scalar/GuardWidening.h index d08d042ab055..fa03d5f678fd 100644 --- a/llvm/include/llvm/Transforms/Scalar/GuardWidening.h +++ b/llvm/include/llvm/Transforms/Scalar/GuardWidening.h @@ -15,12 +15,13 @@ #ifndef LLVM_TRANSFORMS_SCALAR_GUARDWIDENING_H #define LLVM_TRANSFORMS_SCALAR_GUARDWIDENING_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; class Function; struct GuardWideningPass : public PassInfoMixin<GuardWideningPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h b/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h index a1f20d9ca983..4136c45e1905 100644 --- a/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h +++ b/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h @@ -9,11 +9,13 @@ #ifndef LLVM_TRANSFORMS_SCALAR_IVUSERSPRINTER_H #define LLVM_TRANSFORMS_SCALAR_IVUSERSPRINTER_H -#include "llvm/Analysis/IVUsers.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" +#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/IR/PassManager.h" namespace llvm { +class LPMUpdater; +class Loop; +class raw_ostream; /// Printer pass for the \c IVUsers for a loop. class IVUsersPrinterPass : public PassInfoMixin<IVUsersPrinterPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h index 0ac7d7c62b7a..09d08bf423a6 100644 --- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h +++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h @@ -16,14 +16,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/IR/ValueHandle.h" -#include <memory> #include <utility> namespace llvm { @@ -95,10 +92,9 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> { unsigned BBDupThreshold; unsigned DefaultBBDupThreshold; - bool InsertFreezeWhenUnfoldingSelect; public: - JumpThreadingPass(bool InsertFreezeWhenUnfoldingSelect = false, int T = -1); + JumpThreadingPass(int T = -1); // Glue for old PM. bool runImpl(Function &F, TargetLibraryInfo *TLI, TargetTransformInfo *TTI, diff --git a/llvm/include/llvm/Transforms/Scalar/LICM.h b/llvm/include/llvm/Transforms/Scalar/LICM.h index 751f75c0ccb2..f7dd40be47e5 100644 --- a/llvm/include/llvm/Transforms/Scalar/LICM.h +++ b/llvm/include/llvm/Transforms/Scalar/LICM.h @@ -32,46 +32,70 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LICM_H #define LLVM_TRANSFORMS_SCALAR_LICM_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; +class LoopNest; + extern cl::opt<unsigned> SetLicmMssaOptCap; extern cl::opt<unsigned> SetLicmMssaNoAccForPromotionCap; +struct LICMOptions { + unsigned MssaOptCap; + unsigned MssaNoAccForPromotionCap; + bool AllowSpeculation; + + LICMOptions() + : MssaOptCap(SetLicmMssaOptCap), + MssaNoAccForPromotionCap(SetLicmMssaNoAccForPromotionCap), + AllowSpeculation(true) {} + + LICMOptions(unsigned MssaOptCap, unsigned MssaNoAccForPromotionCap, + bool AllowSpeculation) + : MssaOptCap(MssaOptCap), + MssaNoAccForPromotionCap(MssaNoAccForPromotionCap), + AllowSpeculation(AllowSpeculation) {} +}; + /// Performs Loop Invariant Code Motion Pass. class LICMPass : public PassInfoMixin<LICMPass> { - unsigned LicmMssaOptCap; - unsigned LicmMssaNoAccForPromotionCap; + LICMOptions Opts; public: - LICMPass() - : LicmMssaOptCap(SetLicmMssaOptCap), - LicmMssaNoAccForPromotionCap(SetLicmMssaNoAccForPromotionCap) {} - LICMPass(unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap) - : LicmMssaOptCap(LicmMssaOptCap), - LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {} + LICMPass(unsigned MssaOptCap, unsigned MssaNoAccForPromotionCap, + bool AllowSpeculation) + : LICMPass(LICMOptions(MssaOptCap, MssaNoAccForPromotionCap, + AllowSpeculation)) {} + LICMPass(LICMOptions Opts) : Opts(Opts) {} + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); + + void printPipeline(raw_ostream &OS, + function_ref<StringRef(StringRef)> MapClassName2PassName); }; /// Performs LoopNest Invariant Code Motion Pass. class LNICMPass : public PassInfoMixin<LNICMPass> { - unsigned LicmMssaOptCap; - unsigned LicmMssaNoAccForPromotionCap; + LICMOptions Opts; public: - LNICMPass() - : LicmMssaOptCap(SetLicmMssaOptCap), - LicmMssaNoAccForPromotionCap(SetLicmMssaNoAccForPromotionCap) {} - LNICMPass(unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap) - : LicmMssaOptCap(LicmMssaOptCap), - LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {} + LNICMPass(unsigned MssaOptCap, unsigned MssaNoAccForPromotionCap, + bool AllowSpeculation) + : LNICMPass(LICMOptions(MssaOptCap, MssaNoAccForPromotionCap, + AllowSpeculation)) {} + LNICMPass(LICMOptions Opts) : Opts(Opts) {} + PreservedAnalyses run(LoopNest &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); + + void printPipeline(raw_ostream &OS, + function_ref<StringRef(StringRef)> MapClassName2PassName); }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h b/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h index 3f250fc1ce8c..50a837acf4e3 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h @@ -8,12 +8,14 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPACCESSANALYSISPRINTER_H #define LLVM_TRANSFORMS_SCALAR_LOOPACCESSANALYSISPRINTER_H - -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" +#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/IR/PassManager.h" namespace llvm { +class LPMUpdater; +class Loop; +class raw_ostream; /// Printer pass for the \c LoopAccessInfo results. class LoopAccessInfoPrinterPass : public PassInfoMixin<LoopAccessInfoPrinterPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopBoundSplit.h b/llvm/include/llvm/Transforms/Scalar/LoopBoundSplit.h index 306b6fa046df..0c597bf295b2 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopBoundSplit.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopBoundSplit.h @@ -10,11 +10,11 @@ #define LLVM_TRANSFORMS_SCALAR_LOOPBOUNDSPLIT_H #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; /// This pass transforms loops that contain a conditional branch with induction /// variable. For example, it transforms left code to right code: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopDataPrefetch.h b/llvm/include/llvm/Transforms/Scalar/LoopDataPrefetch.h index 9ebd5984cea9..d5e15ffff075 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopDataPrefetch.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopDataPrefetch.h @@ -13,11 +13,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPDATAPREFETCH_H #define LLVM_TRANSFORMS_SCALAR_LOOPDATAPREFETCH_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// An optimization pass inserting data prefetches in loops. class LoopDataPrefetchPass : public PassInfoMixin<LoopDataPrefetchPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopDeletion.h b/llvm/include/llvm/Transforms/Scalar/LoopDeletion.h index 557616e2e6ba..459a5cd3ece4 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopDeletion.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopDeletion.h @@ -14,13 +14,13 @@ #define LLVM_TRANSFORMS_SCALAR_LOOPDELETION_H #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Loop; +class LPMUpdater; + class LoopDeletionPass : public PassInfoMixin<LoopDeletionPass> { public: LoopDeletionPass() = default; diff --git a/llvm/include/llvm/Transforms/Scalar/LoopFlatten.h b/llvm/include/llvm/Transforms/Scalar/LoopFlatten.h index 3d259bdbe986..311b843e83b5 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopFlatten.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopFlatten.h @@ -14,11 +14,11 @@ #define LLVM_TRANSFORMS_SCALAR_LOOPFLATTEN_H #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class LoopNest; class LoopFlattenPass : public PassInfoMixin<LoopFlattenPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopInterchange.h b/llvm/include/llvm/Transforms/Scalar/LoopInterchange.h index c67a30293d2f..8fa14d747f5c 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopInterchange.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopInterchange.h @@ -9,11 +9,14 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPINTERCHANGE_H #define LLVM_TRANSFORMS_SCALAR_LOOPINTERCHANGE_H +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class LoopNest; + struct LoopInterchangePass : public PassInfoMixin<LoopInterchangePass> { PreservedAnalyses run(LoopNest &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h index e83cc2b9bef0..1df510474ca7 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -40,8 +40,6 @@ #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopNestAnalysis.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManager.h" #include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LoopSimplify.h" @@ -52,6 +50,7 @@ namespace llvm { // Forward declarations of an update tracking API used in the pass manager. class LPMUpdater; +class PassInstrumentation; namespace { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPredication.h b/llvm/include/llvm/Transforms/Scalar/LoopPredication.h index 252daafab7a3..83f533603419 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopPredication.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPredication.h @@ -14,12 +14,13 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPPREDICATION_H #define LLVM_TRANSFORMS_SCALAR_LOOPPREDICATION_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; /// Performs Loop Predication Pass. class LoopPredicationPass : public PassInfoMixin<LoopPredicationPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopRotation.h b/llvm/include/llvm/Transforms/Scalar/LoopRotation.h index f68ac70da324..c0e6f105a412 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopRotation.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopRotation.h @@ -13,11 +13,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPROTATION_H #define LLVM_TRANSFORMS_SCALAR_LOOPROTATION_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; /// A simple loop rotation transformation. class LoopRotatePass : public PassInfoMixin<LoopRotatePass> { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h b/llvm/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h index 2d718592aef5..82c8a4406d00 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h @@ -16,12 +16,14 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPSIMPLIFYCFG_H #define LLVM_TRANSFORMS_SCALAR_LOOPSIMPLIFYCFG_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; + /// Performs basic CFG simplifications to assist other loop passes. class LoopSimplifyCFGPass : public PassInfoMixin<LoopSimplifyCFGPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopSink.h b/llvm/include/llvm/Transforms/Scalar/LoopSink.h index 234c48cbebc5..26e50590a625 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopSink.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopSink.h @@ -13,12 +13,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPSINK_H #define LLVM_TRANSFORMS_SCALAR_LOOPSINK_H -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Function; + /// A pass that does profile-guided sinking of instructions into loops. /// /// This is a function pass as it shouldn't be composed into any kind of diff --git a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h index 72663d3d62a8..54f70d7ed4b3 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h @@ -9,10 +9,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPUNROLLANDJAMPASS_H #define LLVM_TRANSFORMS_SCALAR_LOOPUNROLLANDJAMPASS_H +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class LoopNest; /// A simple loop rotation transformation. class LoopUnrollAndJamPass : public PassInfoMixin<LoopUnrollAndJamPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h b/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h index 87d6d6759db2..04e0012330da 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h @@ -9,10 +9,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPVERSIONINGLICM_H #define LLVM_TRANSFORMS_SCALAR_LOOPVERSIONINGLICM_H +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; class LoopVersioningLICMPass : public PassInfoMixin<LoopVersioningLICMPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LowerAtomic.h b/llvm/include/llvm/Transforms/Scalar/LowerAtomicPass.h index 87d945d06901..60bbf916fced 100644 --- a/llvm/include/llvm/Transforms/Scalar/LowerAtomic.h +++ b/llvm/include/llvm/Transforms/Scalar/LowerAtomicPass.h @@ -1,4 +1,4 @@ -//===- LowerAtomic.cpp - Lower atomic intrinsics ----------------*- C++ -*-===// +//===- LowerAtomicPass.h - Lower atomic intrinsics --------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,8 +11,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H -#define LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H +#define LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H #include "llvm/IR/PassManager.h" @@ -25,11 +25,6 @@ public: static bool isRequired() { return true; } }; -class AtomicRMWInst; -/// Convert the given RMWI into primitive load and stores, -/// assuming that doing so is legal. Return true if the lowering -/// succeeds. -bool lowerAtomicRMWInst(AtomicRMWInst *RMWI); } -#endif // LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H +#endif // LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H diff --git a/llvm/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h b/llvm/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h index 61c7bf0454e1..e8e404bb93d6 100644 --- a/llvm/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h +++ b/llvm/include/llvm/Transforms/Scalar/LowerConstantIntrinsics.h @@ -15,11 +15,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOWERCONSTANTINTRINSICS_H #define LLVM_TRANSFORMS_SCALAR_LOWERCONSTANTINTRINSICS_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct LowerConstantIntrinsicsPass : PassInfoMixin<LowerConstantIntrinsicsPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h index 4e47ff70d557..95ef0f73e8af 100644 --- a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h +++ b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h @@ -15,11 +15,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H #define LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> { /// Run the pass over the function. /// diff --git a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h index 3a4db13d670a..8103b0a92489 100644 --- a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h +++ b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h @@ -16,8 +16,6 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/PassManager.h" -#include <cstdint> -#include <functional> namespace llvm { @@ -63,7 +61,7 @@ private: bool processMemMove(MemMoveInst *M); bool performCallSlotOptzn(Instruction *cpyLoad, Instruction *cpyStore, Value *cpyDst, Value *cpySrc, TypeSize cpyLen, - Align cpyAlign, CallInst *C); + Align cpyAlign, std::function<CallInst *()> GetC); bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep); bool processMemSetMemCpyDependence(MemCpyInst *MemCpy, MemSetInst *MemSet); bool performMemCpyToMemSetOptzn(MemCpyInst *MemCpy, MemSetInst *MemSet); diff --git a/llvm/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h b/llvm/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h index 256d03675a07..71e11e59a471 100644 --- a/llvm/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h +++ b/llvm/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h @@ -23,10 +23,11 @@ #ifndef LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H #define LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H -#include "llvm/IR/Module.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; struct MergedLoadStoreMotionOptions { bool SplitFooterBB; MergedLoadStoreMotionOptions(bool SplitFooterBB = false) diff --git a/llvm/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h b/llvm/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h index fd5a06c5051d..b8a8fcc71e57 100644 --- a/llvm/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h +++ b/llvm/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h @@ -15,10 +15,10 @@ #ifndef LLVM_TRANSFORMS_SCALAR_PARTIALLYINLINELIBCALLS_H #define LLVM_TRANSFORMS_SCALAR_PARTIALLYINLINELIBCALLS_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; class PartiallyInlineLibCallsPass : public PassInfoMixin<PartiallyInlineLibCallsPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/SCCP.h b/llvm/include/llvm/Transforms/Scalar/SCCP.h index cd4100447880..032a9b15fc46 100644 --- a/llvm/include/llvm/Transforms/Scalar/SCCP.h +++ b/llvm/include/llvm/Transforms/Scalar/SCCP.h @@ -20,17 +20,19 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SCCP_H #define LLVM_TRANSFORMS_SCALAR_SCCP_H -#include "llvm/ADT/STLExtras.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Utils/PredicateInfo.h" -#include "llvm/Transforms/Utils/SCCPSolver.h" + +#include <functional> namespace llvm { +class AssumptionCache; +class DataLayout; +class Function; +class Module; +class TargetLibraryInfo; +class TargetTransformInfo; +struct AnalysisResultsForFn; /// This pass performs function-level constant propagation and merging. class SCCPPass : public PassInfoMixin<SCCPPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h index e4002159edbd..5e876fc82ac1 100644 --- a/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h +++ b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h @@ -1,5 +1,5 @@ //===- ScalarizeMaskedMemIntrin.h - Scalarize unsupported masked mem ----===// -// instrinsics +// intrinsics // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h index f4472e699295..5cc67f78e5a2 100644 --- a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h +++ b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h @@ -17,14 +17,33 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SCALARIZER_H #define LLVM_TRANSFORMS_SCALAR_SCALARIZER_H +#include "llvm/ADT/Optional.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Function; +class FunctionPass; + +struct ScalarizerPassOptions { + // These optional booleans correspond 1:1 to cl::opt<bool> options defined in + // Scalarizer.cpp. When the cl::opt are specified, they take precedence. + // When the cl::opt are not specified, the present optional booleans allow to + // override the cl::opt's default values. + llvm::Optional<bool> ScalarizeVariableInsertExtract; + llvm::Optional<bool> ScalarizeLoadStore; +}; + class ScalarizerPass : public PassInfoMixin<ScalarizerPass> { + ScalarizerPassOptions Options; + public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + void setScalarizeVariableInsertExtract(bool Value) { + Options.ScalarizeVariableInsertExtract = Value; + } + void setScalarizeLoadStore(bool Value) { Options.ScalarizeLoadStore = Value; } }; /// Create a legacy pass manager instance of the Scalarizer pass diff --git a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h index dfb1619c7f2a..68c121560b13 100644 --- a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h +++ b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h @@ -9,13 +9,18 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SIMPLELOOPUNSWITCH_H #define LLVM_TRANSFORMS_SCALAR_SIMPLELOOPUNSWITCH_H +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class LPMUpdater; +class Loop; +class Pass; +class StringRef; +class raw_ostream; + /// This pass transforms loops that contain branches or switches on loop- /// invariant conditions to have multiple loops. For example, it turns the left /// into the right code: diff --git a/llvm/include/llvm/Transforms/Scalar/Sink.h b/llvm/include/llvm/Transforms/Scalar/Sink.h index 6cbe964d1580..759153f22853 100644 --- a/llvm/include/llvm/Transforms/Scalar/Sink.h +++ b/llvm/include/llvm/Transforms/Scalar/Sink.h @@ -14,11 +14,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SINK_H #define LLVM_TRANSFORMS_SCALAR_SINK_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// Move instructions into successor blocks when possible. class SinkingPass : public PassInfoMixin<SinkingPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/SpeculativeExecution.h b/llvm/include/llvm/Transforms/Scalar/SpeculativeExecution.h index 41de544e7c9c..0ec2a395f875 100644 --- a/llvm/include/llvm/Transforms/Scalar/SpeculativeExecution.h +++ b/llvm/include/llvm/Transforms/Scalar/SpeculativeExecution.h @@ -62,10 +62,10 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SPECULATIVEEXECUTION_H #define LLVM_TRANSFORMS_SCALAR_SPECULATIVEEXECUTION_H -#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/PassManager.h" namespace llvm { +class TargetTransformInfo; class SpeculativeExecutionPass : public PassInfoMixin<SpeculativeExecutionPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/TLSVariableHoist.h b/llvm/include/llvm/Transforms/Scalar/TLSVariableHoist.h new file mode 100644 index 000000000000..2a1b02b40eeb --- /dev/null +++ b/llvm/include/llvm/Transforms/Scalar/TLSVariableHoist.h @@ -0,0 +1,131 @@ +//==- TLSVariableHoist.h ------ Remove Redundant TLS Loads -------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass identifies/eliminates Redundant TLS Loads if related option is set. +// For example: +// static __thread int x; +// int g(); +// int f(int c) { +// int *px = &x; +// while (c--) +// *px += g(); +// return *px; +// } +// +// will generate Redundant TLS Loads by compiling it with +// clang++ -fPIC -ftls-model=global-dynamic -O2 -S +// +// .LBB0_2: # %while.body +// # =>This Inner Loop Header: Depth=1 +// callq _Z1gv@PLT +// movl %eax, %ebp +// leaq _ZL1x@TLSLD(%rip), %rdi +// callq __tls_get_addr@PLT +// addl _ZL1x@DTPOFF(%rax), %ebp +// movl %ebp, _ZL1x@DTPOFF(%rax) +// addl $-1, %ebx +// jne .LBB0_2 +// jmp .LBB0_3 +// .LBB0_4: # %entry.while.end_crit_edge +// leaq _ZL1x@TLSLD(%rip), %rdi +// callq __tls_get_addr@PLT +// movl _ZL1x@DTPOFF(%rax), %ebp +// +// The Redundant TLS Loads will hurt the performance, especially in loops. +// So we try to eliminate/move them if required by customers, let it be: +// +// # %bb.0: # %entry +// ... +// movl %edi, %ebx +// leaq _ZL1x@TLSLD(%rip), %rdi +// callq __tls_get_addr@PLT +// leaq _ZL1x@DTPOFF(%rax), %r14 +// testl %ebx, %ebx +// je .LBB0_1 +// .LBB0_2: # %while.body +// # =>This Inner Loop Header: Depth=1 +// callq _Z1gv@PLT +// addl (%r14), %eax +// movl %eax, (%r14) +// addl $-1, %ebx +// jne .LBB0_2 +// jmp .LBB0_3 +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_TLSVARIABLEHOIST_H +#define LLVM_TRANSFORMS_SCALAR_TLSVARIABLEHOIST_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class BasicBlock; +class DominatorTree; +class Function; +class GlobalVariable; +class Instruction; + +/// A private "module" namespace for types and utilities used by +/// TLSVariableHoist. These are implementation details and should +/// not be used by clients. +namespace tlshoist { + +/// Keeps track of the user of a TLS variable and the operand index +/// where the variable is used. +struct TLSUser { + Instruction *Inst; + unsigned OpndIdx; + + TLSUser(Instruction *Inst, unsigned Idx) : Inst(Inst), OpndIdx(Idx) {} +}; + +/// Keeps track of a TLS variable candidate and its users. +struct TLSCandidate { + SmallVector<TLSUser, 8> Users; + + /// Add the user to the use list and update the cost. + void addUser(Instruction *Inst, unsigned Idx) { + Users.push_back(TLSUser(Inst, Idx)); + } +}; + +} // end namespace tlshoist + +class TLSVariableHoistPass : public PassInfoMixin<TLSVariableHoistPass> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + // Glue for old PM. + bool runImpl(Function &F, DominatorTree &DT, LoopInfo &LI); + +private: + DominatorTree *DT; + LoopInfo *LI; + + /// Keeps track of TLS variable candidates found in the function. + using TLSCandMapType = MapVector<GlobalVariable *, tlshoist::TLSCandidate>; + TLSCandMapType TLSCandMap; + + void collectTLSCandidates(Function &Fn); + void collectTLSCandidate(Instruction *Inst); + Instruction *getNearestLoopDomInst(BasicBlock *BB, Loop *L); + Instruction *getDomInst(Instruction *I1, Instruction *I2); + BasicBlock::iterator findInsertPos(Function &Fn, GlobalVariable *GV, + BasicBlock *&PosBB); + Instruction *genBitCastInst(Function &Fn, GlobalVariable *GV); + bool tryReplaceTLSCandidates(Function &Fn); + bool tryReplaceTLSCandidate(Function &Fn, GlobalVariable *GV); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_TLSVARIABLEHOIST_H diff --git a/llvm/include/llvm/Transforms/Scalar/TailRecursionElimination.h b/llvm/include/llvm/Transforms/Scalar/TailRecursionElimination.h index 906867644504..57b1ed9bf4fe 100644 --- a/llvm/include/llvm/Transforms/Scalar/TailRecursionElimination.h +++ b/llvm/include/llvm/Transforms/Scalar/TailRecursionElimination.h @@ -52,11 +52,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_TAILRECURSIONELIMINATION_H #define LLVM_TRANSFORMS_SCALAR_TAILRECURSIONELIMINATION_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct TailCallElimPass : PassInfoMixin<TailCallElimPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h b/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h index 64691d68b1c4..80d098a1ea52 100644 --- a/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h +++ b/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h @@ -14,10 +14,11 @@ #define LLVM_TRANSFORMS_SCALAR_WARNMISSEDTRANSFORMS_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { class Function; +class Pass; +class PassRegistry; // New pass manager boilerplate. class WarnMissedTransformationsPass diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h index 1e9c0a040ad2..ebd4bd318573 100644 --- a/llvm/include/llvm/Transforms/Utils.h +++ b/llvm/include/llvm/Transforms/Utils.h @@ -155,6 +155,12 @@ FunctionPass *createAssumeSimplifyPass(); // don't block SCEV. // Pass *createCanonicalizeFreezeInLoopsPass(); + +//===----------------------------------------------------------------------===// +// LowerGlobalDtorsLegacy - Lower @llvm.global_dtors by creating wrapper +// functions that are registered in @llvm.global_ctors and which contain a call +// to `__cxa_atexit` to register their destructor functions. +ModulePass *createLowerGlobalDtorsLegacyPass(); } // namespace llvm #endif diff --git a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h index d679bca69510..991ecb8efbd0 100644 --- a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h +++ b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h @@ -17,12 +17,13 @@ #define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H #include "llvm/Analysis/AssumeBundleQueries.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class AssumeInst; +class Function; +class FunctionPass; +class Instruction; class AssumptionCache; class DominatorTree; diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index d99b2a56559d..fcdd2aa0e060 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -18,21 +18,20 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" -#include "llvm/Analysis/DomTreeUpdater.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CFG.h" -#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Dominators.h" #include <cassert> namespace llvm { - +class BranchInst; +class LandingPadInst; +class Loop; +class PHINode; +template <typename PtrType> class SmallPtrSetImpl; class BlockFrequencyInfo; class BranchProbabilityInfo; -class DominatorTree; class DomTreeUpdater; class Function; -class Instruction; class LoopInfo; class MDNode; class MemoryDependenceResults; @@ -500,7 +499,9 @@ BranchInst *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, // create the following structure: // A -> D0A, B -> D0A, I -> D0B, D0A -> D1, D0B -> D1 // If BPI and BFI aren't non-null, BPI/BFI will be updated accordingly. -bool SplitIndirectBrCriticalEdges(Function &F, +// When `IgnoreBlocksWithoutPHI` is set to `true` critical edges leading to a +// block without phi-instructions will not be split. +bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr); diff --git a/llvm/include/llvm/Transforms/Utils/BreakCriticalEdges.h b/llvm/include/llvm/Transforms/Utils/BreakCriticalEdges.h index 3644f1ed7a13..6de080ce3128 100644 --- a/llvm/include/llvm/Transforms/Utils/BreakCriticalEdges.h +++ b/llvm/include/llvm/Transforms/Utils/BreakCriticalEdges.h @@ -17,10 +17,11 @@ #ifndef LLVM_TRANSFORMS_UTILS_BREAKCRITICALEDGES_H #define LLVM_TRANSFORMS_UTILS_BREAKCRITICALEDGES_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { + +class Function; struct BreakCriticalEdgesPass : public PassInfoMixin<BreakCriticalEdgesPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h index 87d33b9b11b7..6ea195ce31ac 100644 --- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h +++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h @@ -22,23 +22,63 @@ namespace llvm { class IRBuilderBase; /// Analyze the name and prototype of the given function and set any - /// applicable attributes. + /// applicable attributes. Note that this merely helps optimizations on an + /// already existing function but does not consider mandatory attributes. + /// /// If the library function is unavailable, this doesn't modify it. /// /// Returns true if any attributes were set and false otherwise. - bool inferLibFuncAttributes(Function &F, const TargetLibraryInfo &TLI); - bool inferLibFuncAttributes(Module *M, StringRef Name, const TargetLibraryInfo &TLI); + bool inferNonMandatoryLibFuncAttrs(Module *M, StringRef Name, + const TargetLibraryInfo &TLI); + bool inferNonMandatoryLibFuncAttrs(Function &F, const TargetLibraryInfo &TLI); + + /// Calls getOrInsertFunction() and then makes sure to add mandatory + /// argument attributes. + FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, + LibFunc TheLibFunc, FunctionType *T, + AttributeList AttributeList); + FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, + LibFunc TheLibFunc, FunctionType *T); + template <typename... ArgsTy> + FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, + LibFunc TheLibFunc, AttributeList AttributeList, + Type *RetTy, ArgsTy... Args) { + SmallVector<Type*, sizeof...(ArgsTy)> ArgTys{Args...}; + return getOrInsertLibFunc(M, TLI, TheLibFunc, + FunctionType::get(RetTy, ArgTys, false), + AttributeList); + } + /// Same as above, but without the attributes. + template <typename... ArgsTy> + FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, + LibFunc TheLibFunc, Type *RetTy, ArgsTy... Args) { + return getOrInsertLibFunc(M, TLI, TheLibFunc, AttributeList{}, RetTy, + Args...); + } + // Avoid an incorrect ordering that'd otherwise compile incorrectly. + template <typename... ArgsTy> + FunctionCallee + getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, + LibFunc TheLibFunc, AttributeList AttributeList, + FunctionType *Invalid, ArgsTy... Args) = delete; + + /// Check whether the library function is available on target and also that + /// it in the current Module is a Function with the right type. + bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI, + LibFunc TheLibFunc); + bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI, + StringRef Name); /// Check whether the overloaded floating point function /// corresponding to \a Ty is available. - bool hasFloatFn(const TargetLibraryInfo *TLI, Type *Ty, + bool hasFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty, LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn); /// Get the name of the overloaded floating point function - /// corresponding to \a Ty. - StringRef getFloatFnName(const TargetLibraryInfo *TLI, Type *Ty, - LibFunc DoubleFn, LibFunc FloatFn, - LibFunc LongDoubleFn); + /// corresponding to \a Ty. Return the LibFunc in \a TheLibFunc. + StringRef getFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty, + LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn, + LibFunc &TheLibFunc); /// Return V if it is an i8*, otherwise cast it to i8*. Value *castToCStr(Value *V, IRBuilderBase &B); @@ -99,6 +139,10 @@ namespace llvm { Value *emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI); + /// Emit a call to the memrchr function, analogously to emitMemChr. + Value *emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, + const DataLayout &DL, const TargetLibraryInfo *TLI); + /// Emit a call to the memcmp function. Value *emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI); @@ -148,7 +192,8 @@ namespace llvm { /// function is known to take a single of type matching 'Op' and returns one /// value with the same type. If 'Op' is a long double, 'l' is added as the /// suffix of name, if 'Op' is a float, we add a 'f' suffix. - Value *emitUnaryFloatFnCall(Value *Op, StringRef Name, IRBuilderBase &B, + Value *emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI, + StringRef Name, IRBuilderBase &B, const AttributeList &Attrs); /// Emit a call to the unary function DoubleFn, FloatFn or LongDoubleFn, @@ -162,8 +207,10 @@ namespace llvm { /// function is known to take type matching 'Op1' and 'Op2' and return one /// value with the same type. If 'Op1/Op2' are long double, 'l' is added as /// the suffix of name, if 'Op1/Op2' are float, we add a 'f' suffix. - Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2, StringRef Name, - IRBuilderBase &B, const AttributeList &Attrs); + Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2, + const TargetLibraryInfo *TLI, + StringRef Name, IRBuilderBase &B, + const AttributeList &Attrs); /// Emit a call to the binary function DoubleFn, FloatFn or LongDoubleFn, /// depending of the type of Op1. diff --git a/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h b/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h index e12d7e09aad6..7e6683fd0c8a 100644 --- a/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h +++ b/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h @@ -16,12 +16,13 @@ #define LLVM_TRANSFORMS_UTILS_CALLGRAPHUPDATER_H #include "llvm/Analysis/CGSCCPassManager.h" -#include "llvm/Analysis/CallGraph.h" -#include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/Analysis/LazyCallGraph.h" namespace llvm { +class CallGraph; +class CallGraphSCC; + /// Wrapper to unify "old style" CallGraph and "new style" LazyCallGraph. This /// simplifies the interface and the call sites, e.g., new and old pass manager /// passes can share the same code. diff --git a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h index daa88981d3bf..fcb384ec3613 100644 --- a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h +++ b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -19,6 +19,7 @@ class CallBase; class CastInst; class Function; class MDNode; +class Value; /// Return true if the given indirect call site can be made to call \p Callee. /// @@ -73,6 +74,15 @@ CallBase &promoteCallWithIfThenElse(CallBase &CB, Function *Callee, /// bool tryPromoteCall(CallBase &CB); +/// Predicate and clone the given call site. +/// +/// This function creates an if-then-else structure at the location of the call +/// site. The "if" condition compares the call site's called value to the given +/// callee. The original call site is moved into the "else" block, and a clone +/// of the call site is placed in the "then" block. The cloned instruction is +/// returned. +CallBase &versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights); + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H diff --git a/llvm/include/llvm/Transforms/Utils/CanonicalizeAliases.h b/llvm/include/llvm/Transforms/Utils/CanonicalizeAliases.h index fdb390db3aff..0bdc1a12d1fb 100644 --- a/llvm/include/llvm/Transforms/Utils/CanonicalizeAliases.h +++ b/llvm/include/llvm/Transforms/Utils/CanonicalizeAliases.h @@ -13,11 +13,12 @@ #ifndef LLVM_TRANSFORMS_UTILS_CANONICALIZEALIASES_H #define LLVM_TRANSFORMS_UTILS_CANONICALIZEALIASES_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// Simple pass that canonicalizes aliases. class CanonicalizeAliasesPass : public PassInfoMixin<CanonicalizeAliasesPass> { public: diff --git a/llvm/include/llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h b/llvm/include/llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h index 9de032935f88..924b6cdf7ca0 100644 --- a/llvm/include/llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h +++ b/llvm/include/llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h @@ -14,10 +14,10 @@ #define LLVM_TRANSFORMS_UTILS_CANONICALIZEFREEZEINLOOPS_H #include "llvm/Analysis/LoopAnalysisManager.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Loop; class LPMUpdater; /// A pass that canonicalizes freeze instructions in a loop. diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h index 8aed3d0e40d9..bb23cf4a9a3c 100644 --- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h +++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h @@ -17,11 +17,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" #include <limits> namespace llvm { +template <typename PtrType> class SmallPtrSetImpl; class AllocaInst; class BasicBlock; class BlockFrequency; @@ -92,6 +92,11 @@ public: BranchProbabilityInfo *BPI; AssumptionCache *AC; + // A block outside of the extraction set where any intermediate + // allocations will be placed inside. If this is null, allocations + // will be placed in the entry block of the function. + BasicBlock *AllocationBlock; + // If true, varargs functions can be extracted. bool AllowVarArgs; @@ -120,11 +125,15 @@ public: /// code is extracted, including vastart. If AllowAlloca is true, then /// extraction of blocks containing alloca instructions would be possible, /// however code extractor won't validate whether extraction is legal. + /// Any new allocations will be placed in the AllocationBlock, unless + /// it is null, in which case it will be placed in the entry block of + /// the function from which the code is being extracted. CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr, bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr, BranchProbabilityInfo *BPI = nullptr, - AssumptionCache *AC = nullptr, - bool AllowVarArgs = false, bool AllowAlloca = false, + AssumptionCache *AC = nullptr, bool AllowVarArgs = false, + bool AllowAlloca = false, + BasicBlock *AllocationBlock = nullptr, std::string Suffix = ""); /// Create a code extractor for a loop body. diff --git a/llvm/include/llvm/Transforms/Utils/CtorUtils.h b/llvm/include/llvm/Transforms/Utils/CtorUtils.h index 3ef3ba244b43..40b290a5a6f4 100644 --- a/llvm/include/llvm/Transforms/Utils/CtorUtils.h +++ b/llvm/include/llvm/Transforms/Utils/CtorUtils.h @@ -13,7 +13,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_CTORUTILS_H #define LLVM_TRANSFORMS_UTILS_CTORUTILS_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace llvm { @@ -22,9 +22,9 @@ class Module; /// Call "ShouldRemove" for every entry in M's global_ctor list and remove the /// entries for which it returns true. Return true if anything changed. -bool optimizeGlobalCtorsList(Module &M, - function_ref<bool(Function *)> ShouldRemove); +bool optimizeGlobalCtorsList( + Module &M, function_ref<bool(uint32_t, Function *)> ShouldRemove); -} // End llvm namespace +} // namespace llvm #endif diff --git a/llvm/include/llvm/Transforms/Utils/Debugify.h b/llvm/include/llvm/Transforms/Utils/Debugify.h index 892e354cd9ed..405bbb8e0be8 100644 --- a/llvm/include/llvm/Transforms/Utils/Debugify.h +++ b/llvm/include/llvm/Transforms/Utils/Debugify.h @@ -23,7 +23,8 @@ #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" -using DebugFnMap = llvm::MapVector<llvm::StringRef, const llvm::DISubprogram *>; +using DebugFnMap = + llvm::MapVector<const llvm::Function *, const llvm::DISubprogram *>; using DebugInstMap = llvm::MapVector<const llvm::Instruction *, bool>; using DebugVarMap = llvm::MapVector<const llvm::DILocalVariable *, unsigned>; using WeakInstValueMap = @@ -42,9 +43,6 @@ struct DebugInfoPerPass { DebugVarMap DIVariables; }; -/// Map pass names to a per-pass DebugInfoPerPass instance. -using DebugInfoPerPassMap = llvm::MapVector<llvm::StringRef, DebugInfoPerPass>; - namespace llvm { class DIBuilder; @@ -69,24 +67,24 @@ bool stripDebugifyMetadata(Module &M); /// /// \param M The module to collect debug information from. /// \param Functions A range of functions to collect debug information from. -/// \param DIPreservationMap A map to collect the DI metadata. +/// \param DebugInfoBeforePass DI metadata before a pass. /// \param Banner A prefix string to add to debug/error messages. /// \param NameOfWrappedPass A name of a pass to add to debug/error messages. bool collectDebugInfoMetadata(Module &M, iterator_range<Module::iterator> Functions, - DebugInfoPerPassMap &DIPreservationMap, + DebugInfoPerPass &DebugInfoBeforePass, StringRef Banner, StringRef NameOfWrappedPass); /// Check original debug information after a pass. /// /// \param M The module to collect debug information from. /// \param Functions A range of functions to collect debug information from. -/// \param DIPreservationMap A map used to check collected the DI metadata. +/// \param DebugInfoBeforePass DI metadata before a pass. /// \param Banner A prefix string to add to debug/error messages. /// \param NameOfWrappedPass A name of a pass to add to debug/error messages. bool checkDebugInfoMetadata(Module &M, iterator_range<Module::iterator> Functions, - DebugInfoPerPassMap &DIPreservationMap, + DebugInfoPerPass &DebugInfoBeforePass, StringRef Banner, StringRef NameOfWrappedPass, StringRef OrigDIVerifyBugsReportFilePath); } // namespace llvm @@ -97,11 +95,11 @@ enum class DebugifyMode { NoDebugify, SyntheticDebugInfo, OriginalDebugInfo }; llvm::ModulePass *createDebugifyModulePass( enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo, llvm::StringRef NameOfWrappedPass = "", - DebugInfoPerPassMap *DIPreservationMap = nullptr); + DebugInfoPerPass *DebugInfoBeforePass = nullptr); llvm::FunctionPass *createDebugifyFunctionPass( enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo, llvm::StringRef NameOfWrappedPass = "", - DebugInfoPerPassMap *DIPreservationMap = nullptr); + DebugInfoPerPass *DebugInfoBeforePass = nullptr); struct NewPMDebugifyPass : public llvm::PassInfoMixin<NewPMDebugifyPass> { llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM); @@ -140,14 +138,14 @@ llvm::ModulePass *createCheckDebugifyModulePass( bool Strip = false, llvm::StringRef NameOfWrappedPass = "", DebugifyStatsMap *StatsMap = nullptr, enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo, - DebugInfoPerPassMap *DIPreservationMap = nullptr, + DebugInfoPerPass *DebugInfoBeforePass = nullptr, llvm::StringRef OrigDIVerifyBugsReportFilePath = ""); llvm::FunctionPass *createCheckDebugifyFunctionPass( bool Strip = false, llvm::StringRef NameOfWrappedPass = "", DebugifyStatsMap *StatsMap = nullptr, enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo, - DebugInfoPerPassMap *DIPreservationMap = nullptr, + DebugInfoPerPass *DebugInfoBeforePass = nullptr, llvm::StringRef OrigDIVerifyBugsReportFilePath = ""); struct NewPMCheckDebugifyPass @@ -171,7 +169,7 @@ struct DebugifyEachInstrumentation { class DebugifyCustomPassManager : public legacy::PassManager { StringRef OrigDIVerifyBugsReportFilePath; DebugifyStatsMap *DIStatsMap = nullptr; - DebugInfoPerPassMap *DIPreservationMap = nullptr; + DebugInfoPerPass *DebugInfoBeforePass = nullptr; enum DebugifyMode Mode = DebugifyMode::NoDebugify; public: @@ -197,17 +195,17 @@ public: // TODO: Implement Debugify for LoopPass. switch (Kind) { case PT_Function: - super::add(createDebugifyFunctionPass(Mode, Name, DIPreservationMap)); + super::add(createDebugifyFunctionPass(Mode, Name, DebugInfoBeforePass)); super::add(P); super::add(createCheckDebugifyFunctionPass( - isSyntheticDebugInfo(), Name, DIStatsMap, Mode, DIPreservationMap, + isSyntheticDebugInfo(), Name, DIStatsMap, Mode, DebugInfoBeforePass, OrigDIVerifyBugsReportFilePath)); break; case PT_Module: - super::add(createDebugifyModulePass(Mode, Name, DIPreservationMap)); + super::add(createDebugifyModulePass(Mode, Name, DebugInfoBeforePass)); super::add(P); super::add(createCheckDebugifyModulePass( - isSyntheticDebugInfo(), Name, DIStatsMap, Mode, DIPreservationMap, + isSyntheticDebugInfo(), Name, DIStatsMap, Mode, DebugInfoBeforePass, OrigDIVerifyBugsReportFilePath)); break; default: @@ -219,8 +217,8 @@ public: // Used within DebugifyMode::SyntheticDebugInfo mode. void setDIStatsMap(DebugifyStatsMap &StatMap) { DIStatsMap = &StatMap; } // Used within DebugifyMode::OriginalDebugInfo mode. - void setDIPreservationMap(DebugInfoPerPassMap &PerPassMap) { - DIPreservationMap = &PerPassMap; + void setDebugInfoBeforePass(DebugInfoPerPass &PerPassDI) { + DebugInfoBeforePass = &PerPassDI; } void setOrigDIVerifyBugsReportFilePath(StringRef BugsReportFilePath) { OrigDIVerifyBugsReportFilePath = BugsReportFilePath; @@ -239,7 +237,7 @@ public: } const DebugifyStatsMap &getDebugifyStatsMap() const { return *DIStatsMap; } - DebugInfoPerPassMap &getDebugInfoPerPassMap() { return *DIPreservationMap; } + DebugInfoPerPass &getDebugInfoPerPass() { return *DebugInfoBeforePass; } }; } // namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h b/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h index bb5c6f04dd0c..3d8447e9bf23 100644 --- a/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h +++ b/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h @@ -32,7 +32,7 @@ class EscapeEnumerator { Function::iterator StateBB, StateE; IRBuilder<> Builder; - bool Done; + bool Done = false; bool HandleExceptions; DomTreeUpdater *DTU; @@ -41,8 +41,7 @@ public: EscapeEnumerator(Function &F, const char *N = "cleanup", bool HandleExceptions = true, DomTreeUpdater *DTU = nullptr) : F(F), CleanupBBName(N), StateBB(F.begin()), StateE(F.end()), - Builder(F.getContext()), Done(false), - HandleExceptions(HandleExceptions), DTU(DTU) {} + Builder(F.getContext()), HandleExceptions(HandleExceptions), DTU(DTU) {} IRBuilder<> *Next(); }; diff --git a/llvm/include/llvm/Transforms/Utils/Evaluator.h b/llvm/include/llvm/Transforms/Utils/Evaluator.h index 99e826bf855f..2b8384897c6b 100644 --- a/llvm/include/llvm/Transforms/Utils/Evaluator.h +++ b/llvm/include/llvm/Transforms/Utils/Evaluator.h @@ -18,8 +18,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Value.h" #include "llvm/Support/Casting.h" #include <cassert> #include <deque> @@ -27,6 +25,7 @@ namespace llvm { +class CallBase; class DataLayout; class Function; class TargetLibraryInfo; @@ -139,6 +138,8 @@ private: SmallVectorImpl<Constant *> &Formals); Constant *ComputeLoadResult(Constant *P, Type *Ty); + Constant *ComputeLoadResult(GlobalVariable *GV, Type *Ty, + const APInt &Offset); /// As we compute SSA register values, we store their contents here. The back /// of the deque contains the current function and the stack contains the diff --git a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h index 964fdce45744..b6b53d0f10cb 100644 --- a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h +++ b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h @@ -16,7 +16,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/IR/Attributes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueMap.h" @@ -28,6 +27,7 @@ namespace llvm { class APFloat; +class AttributeList; class APInt; class BasicBlock; class Constant; diff --git a/llvm/include/llvm/Transforms/Utils/GlobalStatus.h b/llvm/include/llvm/Transforms/Utils/GlobalStatus.h index 775dd23d8f23..60c91fc30174 100644 --- a/llvm/include/llvm/Transforms/Utils/GlobalStatus.h +++ b/llvm/include/llvm/Transforms/Utils/GlobalStatus.h @@ -35,6 +35,9 @@ struct GlobalStatus { /// can be deleted. bool IsLoaded = false; + /// Number of stores to the global. + unsigned NumStores = 0; + /// Keep track of what stores to the global look like. enum StoredType { /// There is no store to this global. It can thus be marked constant. diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h index af9cdb9fd619..d2ce0c5d3988 100644 --- a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h +++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h @@ -18,6 +18,7 @@ #include "llvm/Pass.h" namespace llvm { +class Function; class InjectTLIMappings : public PassInfoMixin<InjectTLIMappings> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index 873127554b47..946fc84b9a2c 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -15,26 +15,18 @@ #define LLVM_TRANSFORMS_UTILS_LOCAL_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/Utils/Local.h" -#include "llvm/IR/Constant.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/User.h" -#include "llvm/IR/Value.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Transforms/Utils/SimplifyCFGOptions.h" #include <cstdint> -#include <limits> namespace llvm { +class DataLayout; +class Value; +class WeakTrackingVH; +class WeakVH; +template <typename T> class SmallVectorImpl; class AAResults; class AllocaInst; class AssumptionCache; @@ -343,7 +335,7 @@ bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, /// Remove all instructions from a basic block other than its terminator /// and any present EH pad instructions. Returns a pair where the first element -/// is the number of instructions (excluding debug info instrinsics) that have +/// is the number of instructions (excluding debug info intrinsics) that have /// been removed, and the second element is the number of debug info intrinsics /// that have been removed. std::pair<unsigned, unsigned> diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 3a712d78df67..676c0c1487db 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -13,18 +13,18 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H -#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/IVDescriptors.h" -#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Transforms/Utils/ValueMapper.h" namespace llvm { template <typename T> class DomTreeNodeBase; using DomTreeNode = DomTreeNodeBase<BasicBlock>; +class StringRef; +class AnalysisUsage; +class TargetTransformInfo; class AAResults; -class AliasSet; -class AliasSetTracker; class BasicBlock; class BlockFrequencyInfo; class ICFLoopSafetyInfo; @@ -49,8 +49,6 @@ typedef std::pair<const RuntimeCheckingPtrGroup *, template <typename T> class Optional; template <typename T, unsigned N> class SmallSetVector; -template <typename T, unsigned N> class SmallVector; -template <typename T> class SmallVectorImpl; template <typename T, unsigned N> class SmallPriorityWorklist; BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, @@ -150,7 +148,7 @@ protected: /// this function is called by \p sinkRegionForLoopNest. bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, BlockFrequencyInfo *, TargetLibraryInfo *, - TargetTransformInfo *, Loop *CurLoop, MemorySSAUpdater *, + TargetTransformInfo *, Loop *CurLoop, MemorySSAUpdater &, ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, Loop *OutermostLoop = nullptr); @@ -159,7 +157,7 @@ bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, BlockFrequencyInfo *, TargetLibraryInfo *, TargetTransformInfo *, Loop *, - MemorySSAUpdater *, ICFLoopSafetyInfo *, + MemorySSAUpdater &, ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *); @@ -171,10 +169,13 @@ bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *, /// BlockFrequencyInfo, TargetLibraryInfo, Loop, AliasSet information for all /// instructions of the loop and loop safety information as arguments. /// Diagnostics is emitted via \p ORE. It returns changed status. +/// \p AllowSpeculation is whether values should be hoisted even if they are not +/// guaranteed to execute in the loop, but are safe to speculatively execute. bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, BlockFrequencyInfo *, TargetLibraryInfo *, Loop *, - MemorySSAUpdater *, ScalarEvolution *, ICFLoopSafetyInfo *, - SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool); + MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *, + SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool, + bool AllowSpeculation); /// This function deletes dead loops. The caller of this function needs to /// guarantee that the loop is infact dead. @@ -204,12 +205,14 @@ void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE, /// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions /// of the loop and loop safety information as arguments. /// Diagnostics is emitted via \p ORE. It returns changed status. +/// \p AllowSpeculation is whether values should be hoisted even if they are not +/// guaranteed to execute in the loop, but are safe to speculatively execute. bool promoteLoopAccessesToScalars( const SmallSetVector<Value *, 8> &, SmallVectorImpl<BasicBlock *> &, SmallVectorImpl<Instruction *> &, SmallVectorImpl<MemoryAccess *> &, PredIteratorCache &, LoopInfo *, DominatorTree *, const TargetLibraryInfo *, - Loop *, MemorySSAUpdater *, ICFLoopSafetyInfo *, - OptimizationRemarkEmitter *); + Loop *, MemorySSAUpdater &, ICFLoopSafetyInfo *, + OptimizationRemarkEmitter *, bool AllowSpeculation); /// Does a BFS from a given node to all of its children inside a given loop. /// The returned vector of nodes includes the starting point. @@ -342,9 +345,9 @@ void getLoopAnalysisUsage(AnalysisUsage &AU); /// true when moving out of loop and not true when moving into loops. /// If \p ORE is set use it to emit optimization remarks. bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT, - Loop *CurLoop, AliasSetTracker *CurAST, - MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop, - SinkAndHoistLICMFlags *LICMFlags = nullptr, + Loop *CurLoop, MemorySSAUpdater &MSSAU, + bool TargetExecutesOncePerLoop, + SinkAndHoistLICMFlags &LICMFlags, OptimizationRemarkEmitter *ORE = nullptr); /// Returns the comparison predicate used when expanding a min/max reduction. @@ -410,8 +413,10 @@ Value *createOrderedReduction(IRBuilderBase &B, /// of each scalar operation (VL) that will be converted into a vector (I). /// If OpValue is non-null, we only consider operations similar to OpValue /// when intersecting. -/// Flag set: NSW, NUW, exact, and all of fast-math. -void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr); +/// Flag set: NSW, NUW (if IncludeWrapFlags is true), exact, and all of +/// fast-math. +void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr, + bool IncludeWrapFlags = true); /// Returns true if we can prove that \p S is defined and always negative in /// loop \p L. @@ -497,6 +502,12 @@ addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl<RuntimePointerCheck> &PointerChecks, SCEVExpander &Expander); +Value * +addDiffRuntimeChecks(Instruction *Loc, Loop *TheLoop, + ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander, + function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, + unsigned IC); + /// Struct to hold information about a partially invariant condition. struct IVConditionInfo { /// Instructions that need to be duplicated and checked for the unswitching diff --git a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h index 4a8831ed45b2..eeab98c56b66 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h +++ b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h @@ -15,7 +15,6 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H #define LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H -#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/PassManager.h" #include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/ValueMapper.h" @@ -23,6 +22,8 @@ namespace llvm { class Loop; +class SCEVPredicate; +class ScalarEvolution; class LoopAccessInfo; class LoopInfo; struct RuntimeCheckingPtrGroup; @@ -113,7 +114,7 @@ private: Loop *VersionedLoop; /// The fall-back loop. I.e. control flows here if pointers in the /// loop may alias (memchecks failed). - Loop *NonVersionedLoop; + Loop *NonVersionedLoop = nullptr; /// This maps the instructions from VersionedLoop to their counterpart /// in NonVersionedLoop. @@ -123,7 +124,7 @@ private: SmallVector<RuntimePointerCheck, 4> AliasChecks; /// The set of SCEV checks that we are versioning for. - const SCEVUnionPredicate &Preds; + const SCEVPredicate &Preds; /// Maps a pointer to the pointer checking group that the pointer /// belongs to. diff --git a/llvm/include/llvm/Transforms/Utils/LowerAtomic.h b/llvm/include/llvm/Transforms/Utils/LowerAtomic.h new file mode 100644 index 000000000000..c85f8e3a5646 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/LowerAtomic.h @@ -0,0 +1,37 @@ +//===- LowerAtomic.h - Lower atomic intrinsics ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +// This pass lowers atomic intrinsics to non-atomic form for use in a known +// non-preemptible environment. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H +#define LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H + +#include "llvm/IR/Instructions.h" + +namespace llvm { + +class IRBuilderBase; + +/// Convert the given Cmpxchg into primitive load and compare. +bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI); + +/// Convert the given RMWI into primitive load and stores, +/// assuming that doing so is legal. Return true if the lowering +/// succeeds. +bool lowerAtomicRMWInst(AtomicRMWInst *RMWI); + +/// Emit IR to implement the given atomicrmw operation on values in registers, +/// returning the new value. +Value *buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, + Value *Loaded, Value *Inc); +} + +#endif // LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H diff --git a/llvm/include/llvm/Transforms/Utils/LowerGlobalDtors.h b/llvm/include/llvm/Transforms/Utils/LowerGlobalDtors.h new file mode 100644 index 000000000000..993a6f57361c --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/LowerGlobalDtors.h @@ -0,0 +1,28 @@ +//===- LowerGlobalDtors.h - Lower @llvm.global_dtors ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers @llvm.global_dtors by creating wrapper functions that are +// registered in @llvm.global_ctors and which contain a call to `__cxa_atexit` +// to register their destructor functions. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_LOWERGLOBALDTORS_H +#define LLVM_TRANSFORMS_UTILS_LOWERGLOBALDTORS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class LowerGlobalDtorsPass : public PassInfoMixin<LowerGlobalDtorsPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_LOWERGLOBALDTORS_H diff --git a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h index 8d0956033d9f..acf59ff580a4 100644 --- a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h +++ b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h @@ -14,13 +14,17 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H #define LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H +#include "llvm/ADT/Optional.h" + namespace llvm { +class AtomicMemCpyInst; class ConstantInt; class Instruction; class MemCpyInst; class MemMoveInst; class MemSetInst; +class ScalarEvolution; class TargetTransformInfo; class Value; struct Align; @@ -28,10 +32,11 @@ struct Align; /// Emit a loop implementing the semantics of llvm.memcpy where the size is not /// a compile-time constant. Loop will be insterted at \p InsertBefore. void createMemCpyLoopUnknownSize(Instruction *InsertBefore, Value *SrcAddr, - Value *DstAddr, Value *CopyLen, - Align SrcAlign, Align DestAlign, - bool SrcIsVolatile, bool DstIsVolatile, - const TargetTransformInfo &TTI); + Value *DstAddr, Value *CopyLen, Align SrcAlign, + Align DestAlign, bool SrcIsVolatile, + bool DstIsVolatile, bool CanOverlap, + const TargetTransformInfo &TTI, + Optional<unsigned> AtomicSize = None); /// Emit a loop implementing the semantics of an llvm.memcpy whose size is a /// compile time constant. Loop is inserted at \p InsertBefore. @@ -39,10 +44,12 @@ void createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, ConstantInt *CopyLen, Align SrcAlign, Align DestAlign, bool SrcIsVolatile, bool DstIsVolatile, - const TargetTransformInfo &TTI); + bool CanOverlap, const TargetTransformInfo &TTI, + Optional<uint32_t> AtomicCpySize = None); /// Expand \p MemCpy as a loop. \p MemCpy is not deleted. -void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI); +void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI, + ScalarEvolution *SE = nullptr); /// Expand \p MemMove as a loop. \p MemMove is not deleted. void expandMemMoveAsLoop(MemMoveInst *MemMove); @@ -50,6 +57,11 @@ void expandMemMoveAsLoop(MemMoveInst *MemMove); /// Expand \p MemSet as a loop. \p MemSet is not deleted. void expandMemSetAsLoop(MemSetInst *MemSet); +/// Expand \p AtomicMemCpy as a loop. \p AtomicMemCpy is not deleted. +void expandAtomicMemCpyAsLoop(AtomicMemCpyInst *AtomicMemCpy, + const TargetTransformInfo &TTI, + ScalarEvolution *SE); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h new file mode 100644 index 000000000000..a2b85e03897b --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h @@ -0,0 +1,82 @@ +//===- MemoryTaggingSupport.h - helpers for memory tagging implementations ===// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares common infrastructure for HWAddressSanitizer and +// Aarch64StackTagging. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_MEMORYTAGGINGSUPPORT_H +#define LLVM_TRANSFORMS_UTILS_MEMORYTAGGINGSUPPORT_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Support/Alignment.h" + +namespace llvm { +class DominatorTree; +class DbgVariableIntrinsic; +class IntrinsicInst; +class PostDominatorTree; +class AllocaInst; +class Instruction; +namespace memtag { +// For an alloca valid between lifetime markers Start and Ends, call the +// Callback for all possible exits out of the lifetime in the containing +// function, which can return from the instructions in RetVec. +// +// Returns whether Ends covered all possible exits. If they did not, +// the caller should remove Ends to ensure that work done at the other +// exits does not happen outside of the lifetime. +bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, + const LoopInfo &LI, const Instruction *Start, + const SmallVectorImpl<IntrinsicInst *> &Ends, + const SmallVectorImpl<Instruction *> &RetVec, + llvm::function_ref<void(Instruction *)> Callback); + +bool isStandardLifetime(const SmallVectorImpl<IntrinsicInst *> &LifetimeStart, + const SmallVectorImpl<IntrinsicInst *> &LifetimeEnd, + const DominatorTree *DT, const LoopInfo *LI, + size_t MaxLifetimes); + +Instruction *getUntagLocationIfFunctionExit(Instruction &Inst); + +struct AllocaInfo { + AllocaInst *AI; + SmallVector<IntrinsicInst *, 2> LifetimeStart; + SmallVector<IntrinsicInst *, 2> LifetimeEnd; + SmallVector<DbgVariableIntrinsic *, 2> DbgVariableIntrinsics; +}; + +struct StackInfo { + MapVector<AllocaInst *, AllocaInfo> AllocasToInstrument; + SmallVector<Instruction *, 4> UnrecognizedLifetimes; + SmallVector<Instruction *, 8> RetVec; + bool CallsReturnTwice = false; +}; + +class StackInfoBuilder { +public: + StackInfoBuilder(std::function<bool(const AllocaInst &)> IsInterestingAlloca) + : IsInterestingAlloca(IsInterestingAlloca) {} + + void visit(Instruction &Inst); + StackInfo &get() { return Info; }; + +private: + StackInfo Info; + std::function<bool(const AllocaInst &)> IsInterestingAlloca; +}; + +uint64_t getAllocaSizeInBytes(const AllocaInst &AI); +void alignAndPadAlloca(memtag::AllocaInfo &Info, llvm::Align Align); + +} // namespace memtag +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Transforms/Utils/MisExpect.h b/llvm/include/llvm/Transforms/Utils/MisExpect.h new file mode 100644 index 000000000000..064eeac4c669 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/MisExpect.h @@ -0,0 +1,77 @@ +//===--- MisExpect.h - Check the use of llvm.expect with PGO data ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This contains code to emit diagnostic messages for potentially incorrect +// usage of the llvm.expect intrinsic. This utility extracts the threshold +// values from metadata associated with the instrumented Branch or Switch +// instruction. The threshold values are then used to determine if a diagnostic +// should be emitted. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LLVMContext.h" + +namespace llvm { +namespace misexpect { + +/// checkBackendInstrumentation - compares PGO counters to the thresholds used +/// for llvm.expect and warns if the PGO counters are outside of the expected +/// range. It extracts the expected weights from the MD_prof weights attatched +/// to the instruction, which are are assumed to come from lowered llvm.expect +/// intrinsics. The RealWeights parameter and the extracted expected weights are +/// then passed to verifyMisexpect() for verification +/// +/// \param I The Instruction being checked +/// \param RealWeights A vector of profile weights for each target block +void checkBackendInstrumentation(Instruction &I, + const llvm::ArrayRef<uint32_t> RealWeights); + +/// checkFrontendInstrumentation - compares PGO counters to the thresholds used +/// for llvm.expect and warns if the PGO counters are outside of the expected +/// range. It extracts the expected weights from the MD_prof weights attatched +/// to the instruction, which are are assumed to come from profiling data +/// attached by the frontend prior to llvm.expect intrinsic lowering. The +/// ExpectedWeights parameter and the extracted real weights are then passed to +/// verifyMisexpect() for verification +/// +/// \param I The Instruction being checked +/// \param ExpectedWeights A vector of the expected weights for each target +/// block, this determines the threshold values used when emiting diagnostics +void checkFrontendInstrumentation(Instruction &I, + const ArrayRef<uint32_t> ExpectedWeights); + +/// veryifyMisExpect - compares RealWeights to the thresholds used +/// for llvm.expect and warns if the PGO counters are outside of the expected +/// range. +/// +/// \param I The Instruction being checked +/// \param RealWeights A vector of profile weights from the profile data +/// \param ExpectedWeights A vector of the weights attatch by llvm.expect +void verifyMisExpect(Instruction &I, ArrayRef<uint32_t> RealWeights, + const ArrayRef<uint32_t> ExpectedWeights); + +/// checkExpectAnnotations - compares PGO counters to the thresholds used +/// for llvm.expect and warns if the PGO counters are outside of the expected +/// range. It extracts the expected weights from the MD_prof weights attatched +/// to the instruction, which are are assumed to come from lowered llvm.expect +/// intrinsics. The RealWeights parameter and the extracted expected weights are +/// then passed to verifyMisexpect() for verification. It is a thin wrapper +/// around the checkFrontendInstrumentation and checkBackendInstrumentation APIs +/// +/// \param I The Instruction being checked +/// \param RealWeights A vector of profile weights for each target block +/// \param IsBackend A boolean describing if this is Frontend instrumentation +void checkExpectAnnotations(Instruction &I, + const ArrayRef<uint32_t> ExistingWeights, + bool IsFrontend); + +} // namespace misexpect +} // namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h index 8d459972336b..85263fc00bc3 100644 --- a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -13,12 +13,13 @@ #ifndef LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/MemoryBufferRef.h" #include <utility> // for std::pair namespace llvm { +template <typename T> class SmallVectorImpl; template <typename T> class ArrayRef; class Module; @@ -109,14 +110,14 @@ std::string getUniqueModuleId(Module *M); /// Embed the memory buffer \p Buf into the module \p M as a global using the /// specified section name. -void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName); +void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName, + Align Alignment = Align(1)); class CallInst; namespace VFABI { /// Overwrite the Vector Function ABI variants attribute with the names provide /// in \p VariantMappings. -void setVectorVariantNames(CallInst *CI, - const SmallVector<std::string, 8> &VariantMappings); +void setVectorVariantNames(CallInst *CI, ArrayRef<std::string> VariantMappings); } // End VFABI namespace } // End llvm namespace diff --git a/llvm/include/llvm/Transforms/Utils/NameAnonGlobals.h b/llvm/include/llvm/Transforms/Utils/NameAnonGlobals.h index 03d8840a22d2..a59f9bc3ebfb 100644 --- a/llvm/include/llvm/Transforms/Utils/NameAnonGlobals.h +++ b/llvm/include/llvm/Transforms/Utils/NameAnonGlobals.h @@ -14,7 +14,6 @@ #ifndef LLVM_TRANSFORMS_UTILS_NAMEANONGLOBALS_H #define LLVM_TRANSFORMS_UTILS_NAMEANONGLOBALS_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h index c922476ac79d..e57e598b6918 100644 --- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h +++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h @@ -56,7 +56,6 @@ #include "llvm/ADT/ilist_node.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" @@ -65,6 +64,7 @@ namespace llvm { class AssumptionCache; class DominatorTree; class Function; +class Value; class IntrinsicInst; class raw_ostream; diff --git a/llvm/include/llvm/Transforms/Utils/RelLookupTableConverter.h b/llvm/include/llvm/Transforms/Utils/RelLookupTableConverter.h index 54c257383fb5..0992a4456c9d 100644 --- a/llvm/include/llvm/Transforms/Utils/RelLookupTableConverter.h +++ b/llvm/include/llvm/Transforms/Utils/RelLookupTableConverter.h @@ -51,11 +51,12 @@ #ifndef LLVM_TRANSFORMS_UTILS_RELLOOKUPTABLECONVERTER_H #define LLVM_TRANSFORMS_UTILS_RELLOOKUPTABLECONVERTER_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + // Pass that converts lookup tables to relative lookup tables. class RelLookupTableConverterPass : public PassInfoMixin<RelLookupTableConverterPass> { diff --git a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h index bf418e659a04..17bd072598ee 100644 --- a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h +++ b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h @@ -16,16 +16,25 @@ #include "llvm/ADT/MapVector.h" #include "llvm/Analysis/DomTreeUpdater.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/ValueLattice.h" -#include "llvm/Analysis/ValueLatticeUtils.h" -#include "llvm/IR/InstVisitor.h" #include "llvm/Transforms/Utils/PredicateInfo.h" -#include <cassert> -#include <utility> #include <vector> namespace llvm { +class Argument; +class BasicBlock; +class CallInst; +class Constant; +class DataLayout; +class DominatorTree; +class Function; +class GlobalVariable; +class Instruction; +class LLVMContext; +class PostDominatorTree; +class StructType; +class TargetLibraryInfo; +class Value; +class ValueLatticeElement; /// Helper struct for bundling up the analysis results per function for IPSCCP. struct AnalysisResultsForFn { @@ -34,6 +43,14 @@ struct AnalysisResultsForFn { PostDominatorTree *PDT; }; +/// Helper struct shared between Function Specialization and SCCP Solver. +struct ArgInfo { + Argument *Formal; // The Formal argument being analysed. + Constant *Actual; // A corresponding actual constant argument. + + ArgInfo(Argument *F, Constant *A) : Formal(F), Actual(A){}; +}; + class SCCPInstVisitor; //===----------------------------------------------------------------------===// @@ -134,11 +151,14 @@ public: /// Return a reference to the set of argument tracked functions. SmallPtrSetImpl<Function *> &getArgumentTrackedFunctions(); - /// Mark argument \p A constant with value \p C in a new function - /// specialization. The argument's parent function is a specialization of the - /// original function \p F. All other arguments of the specialization inherit - /// the lattice state of their corresponding values in the original function. - void markArgInFuncSpecialization(Function *F, Argument *A, Constant *C); + /// Mark the constant arguments of a new function specialization. \p F points + /// to the cloned function and \p Args contains a list of constant arguments + /// represented as pairs of {formal,actual} values (the formal argument is + /// associated with the original function definition). All other arguments of + /// the specialization inherit the lattice state of their corresponding values + /// in the original function. + void markArgInFuncSpecialization(Function *F, + const SmallVectorImpl<ArgInfo> &Args); /// Mark all of the blocks in function \p F non-executable. Clients can used /// this method to erase a function from the module (e.g., if it has been diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h index ee06893ca660..a3e5ac3ac19d 100644 --- a/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h +++ b/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h @@ -323,6 +323,28 @@ public: } while (Changed); } + /// Check all predecessors and if all of them have the same AvailableVal use + /// it as value for block represented by Info. Return true if singluar value + /// is found. + bool FindSingularVal(BBInfo *Info) { + if (!Info->NumPreds) + return false; + ValT Singular = Info->Preds[0]->DefBB->AvailableVal; + if (!Singular) + return false; + for (unsigned Idx = 1; Idx < Info->NumPreds; ++Idx) { + ValT PredVal = Info->Preds[Idx]->DefBB->AvailableVal; + if (!PredVal || Singular != PredVal) + return false; + } + // Record Singular value. + (*AvailableVals)[Info->BB] = Singular; + assert(BBMap[Info->BB] == Info && "Info missed in BBMap?"); + Info->AvailableVal = Singular; + Info->DefBB = Info->Preds[0]->DefBB; + return true; + } + /// FindAvailableVal - If this block requires a PHI, first check if an /// existing PHI matches the PHI placement and reaching definitions computed /// earlier, and if not, create a new PHI. Visit all the block's @@ -339,6 +361,10 @@ public: if (Info->DefBB != Info) continue; + // Look for singular value. + if (FindSingularVal(Info)) + continue; + // Look for an existing PHI. FindExistingPHI(Info->BB, BlockList); if (Info->AvailableVal) diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h index e1f681bbd367..5a4c28063a1d 100644 --- a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h +++ b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h @@ -24,7 +24,6 @@ namespace llvm { -class BasicBlock; class Function; class MachineBasicBlock; class MachineFunction; diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h index 175bdde7fd05..2250e928d1e6 100644 --- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h +++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h @@ -76,6 +76,7 @@ template <> struct IRTraits<BasicBlock> { } // end namespace afdo_detail extern cl::opt<bool> SampleProfileUseProfi; +extern cl::opt<bool> SampleProfileInferEntryCount; template <typename BT> class SampleProfileLoaderBaseImpl { public: @@ -920,7 +921,9 @@ void SampleProfileLoaderBaseImpl<BT>::finalizeWeightPropagation( // Samples->getHeadSamples() + 1 to avoid functions with zero count. if (SampleProfileUseProfi) { const BasicBlockT *EntryBB = getEntryBB(&F); - if (BlockWeights[EntryBB] > 0) { + ErrorOr<uint64_t> EntryWeight = getBlockWeight(EntryBB); + if (BlockWeights[EntryBB] > 0 && + (SampleProfileInferEntryCount || !EntryWeight)) { getFunction(F).setEntryCount( ProfileCount(BlockWeights[EntryBB], Function::PCT_Real), &InlinedGUIDs); diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h index a621cb3078c5..bd7175aa96ff 100644 --- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h +++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h @@ -16,20 +16,14 @@ #define LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERBASEUTIL_H #include "llvm/ADT/DenseMap.h" -#include "llvm/Analysis/ProfileSummaryInfo.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CFG.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DebugLoc.h" -#include "llvm/IR/Function.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Transforms/Utils/ModuleUtils.h" namespace llvm { using namespace sampleprof; class ProfileSummaryInfo; +class Module; extern cl::opt<unsigned> SampleProfileMaxPropagateIterations; extern cl::opt<unsigned> SampleProfileRecordCoverage; diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index 277eb7acf238..260ed1a97831 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -15,13 +15,10 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/InstSimplifyFolder.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ScalarEvolutionNormalization.h" -#include "llvm/Analysis/TargetFolder.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/ValueHandle.h" @@ -293,8 +290,9 @@ public: Value *expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc); /// A specialized variant of expandCodeForPredicate, handling the case when - /// we are expanding code for a SCEVEqualPredicate. - Value *expandEqualPredicate(const SCEVEqualPredicate *Pred, Instruction *Loc); + /// we are expanding code for a SCEVComparePredicate. + Value *expandComparePredicate(const SCEVComparePredicate *Pred, + Instruction *Loc); /// Generates code that evaluates if the \p AR expression will overflow. Value *generateOverflowCheck(const SCEVAddRecExpr *AR, Instruction *Loc, @@ -384,8 +382,8 @@ public: /// Note that this function does not perform an exhaustive search. I.e if it /// didn't find any value it does not mean that there is no such value. /// - Optional<ScalarEvolution::ValueOffsetPair> - getRelatedExistingExpansion(const SCEV *S, const Instruction *At, Loop *L); + Value *getRelatedExistingExpansion(const SCEV *S, const Instruction *At, + Loop *L); /// Returns a suitable insert point after \p I, that dominates \p /// MustDominate. Skips instructions inserted by the expander. @@ -443,21 +441,15 @@ private: Value *expandAddToGEP(const SCEV *Op, PointerType *PTy, Type *Ty, Value *V); /// Find a previous Value in ExprValueMap for expand. - ScalarEvolution::ValueOffsetPair - FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt); + Value *FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt); Value *expand(const SCEV *S); /// Determine the most "relevant" loop for the given SCEV. const Loop *getRelevantLoop(const SCEV *); - Value *expandSMaxExpr(const SCEVNAryExpr *S); - - Value *expandUMaxExpr(const SCEVNAryExpr *S); - - Value *expandSMinExpr(const SCEVNAryExpr *S); - - Value *expandUMinExpr(const SCEVNAryExpr *S); + Value *expandMinMaxExpr(const SCEVNAryExpr *S, Intrinsic::ID IntrinID, + Twine Name, bool IsSequential = false); Value *visitConstant(const SCEVConstant *S) { return S->getValue(); } diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h index fb3a7490346f..7af879638a4d 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h @@ -23,6 +23,7 @@ class AssumptionCache; struct SimplifyCFGOptions { int BonusInstThreshold = 1; bool ForwardSwitchCondToPhi = false; + bool ConvertSwitchRangeToICmp = false; bool ConvertSwitchToLookupTable = false; bool NeedCanonicalLoop = true; bool HoistCommonInsts = false; @@ -41,6 +42,10 @@ struct SimplifyCFGOptions { ForwardSwitchCondToPhi = B; return *this; } + SimplifyCFGOptions &convertSwitchRangeToICmp(bool B) { + ConvertSwitchRangeToICmp = B; + return *this; + } SimplifyCFGOptions &convertSwitchToLookupTable(bool B) { ConvertSwitchToLookupTable = B; return *this; diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h index 4ba56fb45afa..ff60811b6168 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -15,12 +15,11 @@ #ifndef LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H #define LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H -#include "llvm/Analysis/ScalarEvolutionExpressions.h" -#include "llvm/IR/ConstantRange.h" -#include "llvm/IR/ValueHandle.h" - namespace llvm { +class Type; +class WeakTrackingVH; +template <typename T> class SmallVectorImpl; class CastInst; class DominatorTree; class Loop; diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h index a88e72fc9ba8..79a44b667445 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -14,7 +14,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_SIMPLIFYLIBCALLS_H #define LLVM_TRANSFORMS_UTILS_SIMPLIFYLIBCALLS_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Analysis/TargetLibraryInfo.h" namespace llvm { @@ -105,7 +105,7 @@ private: OptimizationRemarkEmitter &ORE; BlockFrequencyInfo *BFI; ProfileSummaryInfo *PSI; - bool UnsafeFPShrink; + bool UnsafeFPShrink = false; function_ref<void(Instruction *, Value *)> Replacer; function_ref<void(Instruction *)> Eraser; @@ -163,6 +163,7 @@ private: Value *optimizeStpCpy(CallInst *CI, IRBuilderBase &B); Value *optimizeStrNCpy(CallInst *CI, IRBuilderBase &B); Value *optimizeStrLen(CallInst *CI, IRBuilderBase &B); + Value *optimizeStrNLen(CallInst *CI, IRBuilderBase &B); Value *optimizeStrPBrk(CallInst *CI, IRBuilderBase &B); Value *optimizeStrTo(CallInst *CI, IRBuilderBase &B); Value *optimizeStrSpn(CallInst *CI, IRBuilderBase &B); @@ -234,10 +235,11 @@ private: /// hasFloatVersion - Checks if there is a float version of the specified /// function by checking for an existing function with name FuncName + f - bool hasFloatVersion(StringRef FuncName); + bool hasFloatVersion(const Module *M, StringRef FuncName); - /// Shared code to optimize strlen+wcslen. - Value *optimizeStringLength(CallInst *CI, IRBuilderBase &B, unsigned CharSize); + /// Shared code to optimize strlen+wcslen and strnlen+wcsnlen. + Value *optimizeStringLength(CallInst *CI, IRBuilderBase &B, unsigned CharSize, + Value *Bound = nullptr); }; } // End llvm namespace diff --git a/llvm/include/llvm/Transforms/Utils/SizeOpts.h b/llvm/include/llvm/Transforms/Utils/SizeOpts.h index 11bf5501598f..aa9e9bd6c69b 100644 --- a/llvm/include/llvm/Transforms/Utils/SizeOpts.h +++ b/llvm/include/llvm/Transforms/Utils/SizeOpts.h @@ -13,7 +13,6 @@ #ifndef LLVM_TRANSFORMS_UTILS_SIZEOPTS_H #define LLVM_TRANSFORMS_UTILS_SIZEOPTS_H -#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Support/CommandLine.h" diff --git a/llvm/include/llvm/Transforms/Utils/SplitModule.h b/llvm/include/llvm/Transforms/Utils/SplitModule.h index 42b3784db417..a5450738060a 100644 --- a/llvm/include/llvm/Transforms/Utils/SplitModule.h +++ b/llvm/include/llvm/Transforms/Utils/SplitModule.h @@ -15,7 +15,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_SPLITMODULE_H #define LLVM_TRANSFORMS_UTILS_SPLITMODULE_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include <memory> namespace llvm { diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h index 320c36b36924..65fe8eff6442 100644 --- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h @@ -17,6 +17,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Support/InstructionCost.h" namespace llvm { @@ -123,11 +124,9 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences( Optional<bool> UserAllowPartial, Optional<bool> UserRuntime, Optional<bool> UserUpperBound, Optional<unsigned> UserFullUnrollMaxCount); -unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls, - bool &NotDuplicatable, bool &Convergent, - const TargetTransformInfo &TTI, - const SmallPtrSetImpl<const Value *> &EphValues, - unsigned BEInsns); +InstructionCost ApproximateLoopSize(const Loop *L, unsigned &NumCalls, + bool &NotDuplicatable, bool &Convergent, const TargetTransformInfo &TTI, + const SmallPtrSetImpl<const Value *> &EphValues, unsigned BEInsns); } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h index 3636285e38f5..15a46baa190d 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h @@ -10,9 +10,10 @@ #define LLVM_TRANSFORMS_VECTORIZE_LOADSTOREVECTORIZER_H #include "llvm/IR/PassManager.h" -#include "llvm/Pass.h" namespace llvm { +class Pass; +class Function; class LoadStoreVectorizerPass : public PassInfoMixin<LoadStoreVectorizerPass> { public: diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index 32d295a2dd16..b01bd222b252 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -28,12 +28,26 @@ #include "llvm/ADT/MapVector.h" #include "llvm/Analysis/LoopAccessAnalysis.h" -#include "llvm/Analysis/OptimizationRemarkEmitter.h" -#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Support/TypeSize.h" #include "llvm/Transforms/Utils/LoopUtils.h" namespace llvm { +class AAResults; +class AssumptionCache; +class BasicBlock; +class BlockFrequencyInfo; +class DemandedBits; +class DominatorTree; +class Function; +class Loop; +class LoopInfo; +class Metadata; +class OptimizationRemarkEmitter; +class PredicatedScalarEvolution; +class ProfileSummaryInfo; +class TargetLibraryInfo; +class TargetTransformInfo; +class Type; /// Utility class for getting and setting loop vectorizer hints in the form /// of loop metadata. @@ -207,7 +221,6 @@ public: void addRuntimePointerChecks(unsigned Num) { NumRuntimePointerChecks = Num; } - Instruction *getExactFPInst() { return ExactFPMathInst; } unsigned getNumRuntimePointerChecks() const { @@ -294,6 +307,14 @@ public: /// Returns the widest induction type. Type *getWidestInductionType() { return WidestIndTy; } + /// Returns True if given store is a final invariant store of one of the + /// reductions found in the loop. + bool isInvariantStoreOfReduction(StoreInst *SI); + + /// Returns True if given address is invariant and is used to store recurrent + /// expression + bool isInvariantAddressOfReduction(Value *V); + /// Returns True if V is a Phi node of an induction variable in this loop. bool isInductionPhi(const Value *V) const; @@ -301,6 +322,10 @@ public: /// floating point induction. const InductionDescriptor *getIntOrFpInductionDescriptor(PHINode *Phi) const; + /// Returns a pointer to the induction descriptor, if \p Phi is pointer + /// induction. + const InductionDescriptor *getPointerInductionDescriptor(PHINode *Phi) const; + /// Returns True if V is a cast that is part of an induction def-use chain, /// and had been proven to be redundant under a runtime guard (in other /// words, the cast has the same SCEV expression as the induction phi). diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h index cd605aacb52d..b41f3efc5b55 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -20,7 +20,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/MapVector.h" -#include "llvm/ADT/None.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" @@ -30,7 +29,6 @@ class AAResults; class AssumptionCache; class BasicBlock; class CmpInst; -class DataLayout; class DemandedBits; class DominatorTree; class Function; @@ -135,7 +133,7 @@ private: bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R); bool vectorizeStoreChain(ArrayRef<Value *> Chain, slpvectorizer::BoUpSLP &R, - unsigned Idx); + unsigned Idx, unsigned MinVF); bool vectorizeStores(ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R); diff --git a/llvm/include/llvm/WindowsDriver/MSVCPaths.h b/llvm/include/llvm/WindowsDriver/MSVCPaths.h new file mode 100644 index 000000000000..7256a4f66eaa --- /dev/null +++ b/llvm/include/llvm/WindowsDriver/MSVCPaths.h @@ -0,0 +1,107 @@ +//===-- MSVCPaths.h - MSVC path-parsing helpers -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_MSVCPATHS_H +#define LLVM_SUPPORT_MSVCPATHS_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include <string> + +namespace llvm { + +namespace vfs { +class FileSystem; +} + +enum class SubDirectoryType { + Bin, + Include, + Lib, +}; + +enum class ToolsetLayout { + OlderVS, + VS2017OrNewer, + DevDivInternal, +}; + +// Windows SDKs and VC Toolchains group their contents into subdirectories based +// on the target architecture. This function converts an llvm::Triple::ArchType +// to the corresponding subdirectory name. +const char *archToWindowsSDKArch(llvm::Triple::ArchType Arch); + +// Similar to the above function, but for Visual Studios before VS2017. +const char *archToLegacyVCArch(llvm::Triple::ArchType Arch); + +// Similar to the above function, but for DevDiv internal builds. +const char *archToDevDivInternalArch(llvm::Triple::ArchType Arch); + +bool appendArchToWindowsSDKLibPath(int SDKMajor, llvm::SmallString<128> LibPath, + llvm::Triple::ArchType Arch, + std::string &path); + +// Get the path to a specific subdirectory in the current toolchain for +// a given target architecture. +// VS2017 changed the VC toolchain layout, so this should be used instead +// of hardcoding paths. +std::string getSubDirectoryPath(SubDirectoryType Type, ToolsetLayout VSLayout, + const std::string &VCToolChainPath, + llvm::Triple::ArchType TargetArch, + llvm::StringRef SubdirParent = ""); + +// Check if the Include path of a specified version of Visual Studio contains +// specific header files. If not, they are probably shipped with Universal CRT. +bool useUniversalCRT(ToolsetLayout VSLayout, const std::string &VCToolChainPath, + llvm::Triple::ArchType TargetArch, + llvm::vfs::FileSystem &VFS); + +/// Get Windows SDK installation directory. +bool getWindowsSDKDir(vfs::FileSystem &VFS, + llvm::Optional<llvm::StringRef> WinSdkDir, + llvm::Optional<llvm::StringRef> WinSdkVersion, + llvm::Optional<llvm::StringRef> WinSysRoot, + std::string &Path, int &Major, + std::string &WindowsSDKIncludeVersion, + std::string &WindowsSDKLibVersion); + +bool getUniversalCRTSdkDir(vfs::FileSystem &VFS, + llvm::Optional<llvm::StringRef> WinSdkDir, + llvm::Optional<llvm::StringRef> WinSdkVersion, + llvm::Optional<llvm::StringRef> WinSysRoot, + std::string &Path, + std::string &UCRTVersion); + +// Check command line arguments to try and find a toolchain. +bool findVCToolChainViaCommandLine( + vfs::FileSystem &VFS, llvm::Optional<llvm::StringRef> VCToolsDir, + llvm::Optional<llvm::StringRef> VCToolsVersion, + llvm::Optional<llvm::StringRef> WinSysRoot, std::string &Path, + ToolsetLayout &VSLayout); + +// Check various environment variables to try and find a toolchain. +bool findVCToolChainViaEnvironment(vfs::FileSystem &VFS, std::string &Path, + ToolsetLayout &VSLayout); + +// Query the Setup Config server for installs, then pick the newest version +// and find its default VC toolchain. +// This is the preferred way to discover new Visual Studios, as they're no +// longer listed in the registry. +bool findVCToolChainViaSetupConfig(vfs::FileSystem &VFS, std::string &Path, + ToolsetLayout &VSLayout); + +// Look in the registry for Visual Studio installs, and use that to get +// a toolchain path. VS2017 and newer don't get added to the registry. +// So if we find something here, we know that it's an older version. +bool findVCToolChainViaRegistry(std::string &Path, ToolsetLayout &VSLayout); + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/WindowsDriver/MSVCSetupApi.h b/llvm/include/llvm/WindowsDriver/MSVCSetupApi.h new file mode 100644 index 000000000000..28e6e3e08e37 --- /dev/null +++ b/llvm/include/llvm/WindowsDriver/MSVCSetupApi.h @@ -0,0 +1,523 @@ +// <copyright file="Program.cpp" company="Microsoft Corporation"> +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// </copyright> +// <license> +// The MIT License (MIT) +// +// Copyright (C) Microsoft Corporation. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// </license> + +#pragma once + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#endif + +// Constants +// +#ifndef E_NOTFOUND +#define E_NOTFOUND HRESULT_FROM_WIN32(ERROR_NOT_FOUND) +#endif + +#ifndef E_FILENOTFOUND +#define E_FILENOTFOUND HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) +#endif + +// Enumerations +// +/// <summary> +/// The state of an instance. +/// </summary> +enum InstanceState : unsigned { + /// <summary> + /// The instance state has not been determined. + /// </summary> + eNone = 0, + + /// <summary> + /// The instance installation path exists. + /// </summary> + eLocal = 1, + + /// <summary> + /// A product is registered to the instance. + /// </summary> + eRegistered = 2, + + /// <summary> + /// No reboot is required for the instance. + /// </summary> + eNoRebootRequired = 4, + + /// <summary> + /// The instance represents a complete install. + /// </summary> + eComplete = MAXUINT, +}; + +// Forward interface declarations +// +#ifndef __ISetupInstance_FWD_DEFINED__ +#define __ISetupInstance_FWD_DEFINED__ +typedef struct ISetupInstance ISetupInstance; +#endif + +#ifndef __ISetupInstance2_FWD_DEFINED__ +#define __ISetupInstance2_FWD_DEFINED__ +typedef struct ISetupInstance2 ISetupInstance2; +#endif + +#ifndef __IEnumSetupInstances_FWD_DEFINED__ +#define __IEnumSetupInstances_FWD_DEFINED__ +typedef struct IEnumSetupInstances IEnumSetupInstances; +#endif + +#ifndef __ISetupConfiguration_FWD_DEFINED__ +#define __ISetupConfiguration_FWD_DEFINED__ +typedef struct ISetupConfiguration ISetupConfiguration; +#endif + +#ifndef __ISetupConfiguration2_FWD_DEFINED__ +#define __ISetupConfiguration2_FWD_DEFINED__ +typedef struct ISetupConfiguration2 ISetupConfiguration2; +#endif + +#ifndef __ISetupPackageReference_FWD_DEFINED__ +#define __ISetupPackageReference_FWD_DEFINED__ +typedef struct ISetupPackageReference ISetupPackageReference; +#endif + +#ifndef __ISetupHelper_FWD_DEFINED__ +#define __ISetupHelper_FWD_DEFINED__ +typedef struct ISetupHelper ISetupHelper; +#endif + +// Forward class declarations +// +#ifndef __SetupConfiguration_FWD_DEFINED__ +#define __SetupConfiguration_FWD_DEFINED__ + +#ifdef __cplusplus +typedef class SetupConfiguration SetupConfiguration; +#endif + +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// Interface definitions +// +EXTERN_C const IID IID_ISetupInstance; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// Information about an instance of a product. +/// </summary> +struct DECLSPEC_UUID("B41463C3-8866-43B5-BC33-2B0676F7F42E") + DECLSPEC_NOVTABLE ISetupInstance : public IUnknown { + /// <summary> + /// Gets the instance identifier (should match the name of the parent instance + /// directory). + /// </summary> + /// <param name="pbstrInstanceId">The instance identifier.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist.</returns> + STDMETHOD(GetInstanceId)(_Out_ BSTR *pbstrInstanceId) = 0; + + /// <summary> + /// Gets the local date and time when the installation was originally + /// installed. + /// </summary> + /// <param name="pInstallDate">The local date and time when the installation + /// was originally installed.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetInstallDate)(_Out_ LPFILETIME pInstallDate) = 0; + + /// <summary> + /// Gets the unique name of the installation, often indicating the branch and + /// other information used for telemetry. + /// </summary> + /// <param name="pbstrInstallationName">The unique name of the installation, + /// often indicating the branch and other information used for + /// telemetry.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetInstallationName)(_Out_ BSTR *pbstrInstallationName) = 0; + + /// <summary> + /// Gets the path to the installation root of the product. + /// </summary> + /// <param name="pbstrInstallationPath">The path to the installation root of + /// the product.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetInstallationPath)(_Out_ BSTR *pbstrInstallationPath) = 0; + + /// <summary> + /// Gets the version of the product installed in this instance. + /// </summary> + /// <param name="pbstrInstallationVersion">The version of the product + /// installed in this instance.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetInstallationVersion)(_Out_ BSTR *pbstrInstallationVersion) = 0; + + /// <summary> + /// Gets the display name (title) of the product installed in this instance. + /// </summary> + /// <param name="lcid">The LCID for the display name.</param> + /// <param name="pbstrDisplayName">The display name (title) of the product + /// installed in this instance.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetDisplayName)(_In_ LCID lcid, _Out_ BSTR *pbstrDisplayName) = 0; + + /// <summary> + /// Gets the description of the product installed in this instance. + /// </summary> + /// <param name="lcid">The LCID for the description.</param> + /// <param name="pbstrDescription">The description of the product installed in + /// this instance.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(GetDescription)(_In_ LCID lcid, _Out_ BSTR *pbstrDescription) = 0; + + /// <summary> + /// Resolves the optional relative path to the root path of the instance. + /// </summary> + /// <param name="pwszRelativePath">A relative path within the instance to + /// resolve, or NULL to get the root path.</param> + /// <param name="pbstrAbsolutePath">The full path to the optional relative + /// path within the instance. If the relative path is NULL, the root path will + /// always terminate in a backslash.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// property is not defined.</returns> + STDMETHOD(ResolvePath) + (_In_opt_z_ LPCOLESTR pwszRelativePath, _Out_ BSTR *pbstrAbsolutePath) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupInstance2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// Information about an instance of a product. +/// </summary> +struct DECLSPEC_UUID("89143C9A-05AF-49B0-B717-72E218A2185C") + DECLSPEC_NOVTABLE ISetupInstance2 : public ISetupInstance { + /// <summary> + /// Gets the state of the instance. + /// </summary> + /// <param name="pState">The state of the instance.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist.</returns> + STDMETHOD(GetState)(_Out_ InstanceState *pState) = 0; + + /// <summary> + /// Gets an array of package references registered to the instance. + /// </summary> + /// <param name="ppsaPackages">Pointer to an array of <see + /// cref="ISetupPackageReference"/>.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// packages property is not defined.</returns> + STDMETHOD(GetPackages)(_Out_ LPSAFEARRAY *ppsaPackages) = 0; + + /// <summary> + /// Gets a pointer to the <see cref="ISetupPackageReference"/> that represents + /// the registered product. + /// </summary> + /// <param name="ppPackage">Pointer to an instance of <see + /// cref="ISetupPackageReference"/>. This may be NULL if <see + /// cref="GetState"/> does not return <see cref="eComplete"/>.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the + /// packages property is not defined.</returns> + STDMETHOD(GetProduct) + (_Outptr_result_maybenull_ ISetupPackageReference **ppPackage) = 0; + + /// <summary> + /// Gets the relative path to the product application, if available. + /// </summary> + /// <param name="pbstrProductPath">The relative path to the product + /// application, if available.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_FILENOTFOUND if the instance state does not exist.</returns> + STDMETHOD(GetProductPath) + (_Outptr_result_maybenull_ BSTR *pbstrProductPath) = 0; +}; +#endif + +EXTERN_C const IID IID_IEnumSetupInstances; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// A enumerator of installed <see cref="ISetupInstance"/> objects. +/// </summary> +struct DECLSPEC_UUID("6380BCFF-41D3-4B2E-8B2E-BF8A6810C848") + DECLSPEC_NOVTABLE IEnumSetupInstances : public IUnknown { + /// <summary> + /// Retrieves the next set of product instances in the enumeration sequence. + /// </summary> + /// <param name="celt">The number of product instances to retrieve.</param> + /// <param name="rgelt">A pointer to an array of <see + /// cref="ISetupInstance"/>.</param> + /// <param name="pceltFetched">A pointer to the number of product instances + /// retrieved. If celt is 1 this parameter may be NULL.</param> + /// <returns>S_OK if the number of elements were fetched, S_FALSE if nothing + /// was fetched (at end of enumeration), E_INVALIDARG if celt is greater than + /// 1 and pceltFetched is NULL, or E_OUTOFMEMORY if an <see + /// cref="ISetupInstance"/> could not be allocated.</returns> + STDMETHOD(Next) + (_In_ ULONG celt, _Out_writes_to_(celt, *pceltFetched) ISetupInstance **rgelt, + _Out_opt_ _Deref_out_range_(0, celt) ULONG *pceltFetched) = 0; + + /// <summary> + /// Skips the next set of product instances in the enumeration sequence. + /// </summary> + /// <param name="celt">The number of product instances to skip.</param> + /// <returns>S_OK if the number of elements could be skipped; otherwise, + /// S_FALSE;</returns> + STDMETHOD(Skip)(_In_ ULONG celt) = 0; + + /// <summary> + /// Resets the enumeration sequence to the beginning. + /// </summary> + /// <returns>Always returns S_OK;</returns> + STDMETHOD(Reset)(void) = 0; + + /// <summary> + /// Creates a new enumeration object in the same state as the current + /// enumeration object: the new object points to the same place in the + /// enumeration sequence. + /// </summary> + /// <param name="ppenum">A pointer to a pointer to a new <see + /// cref="IEnumSetupInstances"/> interface. If the method fails, this + /// parameter is undefined.</param> + /// <returns>S_OK if a clone was returned; otherwise, E_OUTOFMEMORY.</returns> + STDMETHOD(Clone)(_Deref_out_opt_ IEnumSetupInstances **ppenum) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupConfiguration; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// Gets information about product instances set up on the machine. +/// </summary> +struct DECLSPEC_UUID("42843719-DB4C-46C2-8E7C-64F1816EFD5B") + DECLSPEC_NOVTABLE ISetupConfiguration : public IUnknown { + /// <summary> + /// Enumerates all completed product instances installed. + /// </summary> + /// <param name="ppEnumInstances">An enumeration of completed, installed + /// product instances.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(EnumInstances)(_Out_ IEnumSetupInstances **ppEnumInstances) = 0; + + /// <summary> + /// Gets the instance for the current process path. + /// </summary> + /// <param name="ppInstance">The instance for the current process + /// path.</param> + /// <returns>The instance for the current process path, or E_NOTFOUND if not + /// found.</returns> + STDMETHOD(GetInstanceForCurrentProcess) + (_Out_ ISetupInstance **ppInstance) = 0; + + /// <summary> + /// Gets the instance for the given path. + /// </summary> + /// <param name="ppInstance">The instance for the given path.</param> + /// <returns>The instance for the given path, or E_NOTFOUND if not + /// found.</returns> + STDMETHOD(GetInstanceForPath) + (_In_z_ LPCWSTR wzPath, _Out_ ISetupInstance **ppInstance) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupConfiguration2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// Gets information about product instances. +/// </summary> +struct DECLSPEC_UUID("26AAB78C-4A60-49D6-AF3B-3C35BC93365D") + DECLSPEC_NOVTABLE ISetupConfiguration2 : public ISetupConfiguration { + /// <summary> + /// Enumerates all product instances. + /// </summary> + /// <param name="ppEnumInstances">An enumeration of all product + /// instances.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(EnumAllInstances)(_Out_ IEnumSetupInstances **ppEnumInstances) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupPackageReference; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// A reference to a package. +/// </summary> +struct DECLSPEC_UUID("da8d8a16-b2b6-4487-a2f1-594ccccd6bf5") + DECLSPEC_NOVTABLE ISetupPackageReference : public IUnknown { + /// <summary> + /// Gets the general package identifier. + /// </summary> + /// <param name="pbstrId">The general package identifier.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetId)(_Out_ BSTR *pbstrId) = 0; + + /// <summary> + /// Gets the version of the package. + /// </summary> + /// <param name="pbstrVersion">The version of the package.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetVersion)(_Out_ BSTR *pbstrVersion) = 0; + + /// <summary> + /// Gets the target process architecture of the package. + /// </summary> + /// <param name="pbstrChip">The target process architecture of the + /// package.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetChip)(_Out_ BSTR *pbstrChip) = 0; + + /// <summary> + /// Gets the language and optional region identifier. + /// </summary> + /// <param name="pbstrLanguage">The language and optional region + /// identifier.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetLanguage)(_Out_ BSTR *pbstrLanguage) = 0; + + /// <summary> + /// Gets the build branch of the package. + /// </summary> + /// <param name="pbstrBranch">The build branch of the package.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetBranch)(_Out_ BSTR *pbstrBranch) = 0; + + /// <summary> + /// Gets the type of the package. + /// </summary> + /// <param name="pbstrType">The type of the package.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(GetType)(_Out_ BSTR *pbstrType) = 0; + + /// <summary> + /// Gets the unique identifier consisting of all defined tokens. + /// </summary> + /// <param name="pbstrUniqueId">The unique identifier consisting of all + /// defined tokens.</param> + /// <returns>Standard HRESULT indicating success or failure, including + /// E_UNEXPECTED if no Id was defined (required).</returns> + STDMETHOD(GetUniqueId)(_Out_ BSTR *pbstrUniqueId) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupHelper; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// <summary> +/// Helper functions. +/// </summary> +/// <remarks> +/// You can query for this interface from the <see cref="SetupConfiguration"/> +/// class. +/// </remarks> +struct DECLSPEC_UUID("42b21b78-6192-463e-87bf-d577838f1d5c") + DECLSPEC_NOVTABLE ISetupHelper : public IUnknown { + /// <summary> + /// Parses a dotted quad version string into a 64-bit unsigned integer. + /// </summary> + /// <param name="pwszVersion">The dotted quad version string to parse, e.g. + /// 1.2.3.4.</param> + /// <param name="pullVersion">A 64-bit unsigned integer representing the + /// version. You can compare this to other versions.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(ParseVersion) + (_In_ LPCOLESTR pwszVersion, _Out_ PULONGLONG pullVersion) = 0; + + /// <summary> + /// Parses a dotted quad version string into a 64-bit unsigned integer. + /// </summary> + /// <param name="pwszVersionRange">The string containing 1 or 2 dotted quad + /// version strings to parse, e.g. [1.0,) that means 1.0.0.0 or newer.</param> + /// <param name="pullMinVersion">A 64-bit unsigned integer representing the + /// minimum version, which may be 0. You can compare this to other + /// versions.</param> + /// <param name="pullMaxVersion">A 64-bit unsigned integer representing the + /// maximum version, which may be MAXULONGLONG. You can compare this to other + /// versions.</param> + /// <returns>Standard HRESULT indicating success or failure.</returns> + STDMETHOD(ParseVersionRange) + (_In_ LPCOLESTR pwszVersionRange, _Out_ PULONGLONG pullMinVersion, + _Out_ PULONGLONG pullMaxVersion) = 0; +}; +#endif + +// Class declarations +// +EXTERN_C const CLSID CLSID_SetupConfiguration; + +#ifdef __cplusplus +/// <summary> +/// This class implements <see cref="ISetupConfiguration"/>, <see +/// cref="ISetupConfiguration2"/>, and <see cref="ISetupHelper"/>. +/// </summary> +class DECLSPEC_UUID("177F0C4A-1CD3-4DE7-A32C-71DBBB9FA36D") SetupConfiguration; +#endif + +// Function declarations +// +/// <summary> +/// Gets an <see cref="ISetupConfiguration"/> that provides information about +/// product instances installed on the machine. +/// </summary> +/// <param name="ppConfiguration">The <see cref="ISetupConfiguration"/> that +/// provides information about product instances installed on the +/// machine.</param> +/// <param name="pReserved">Reserved for future use.</param> +/// <returns>Standard HRESULT indicating success or failure.</returns> +STDMETHODIMP GetSetupConfiguration(_Out_ ISetupConfiguration **ppConfiguration, + _Reserved_ LPVOID pReserved); + +#ifdef __cplusplus +} +#endif + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap index d0693ccfd8f6..76b10621541c 100644 --- a/llvm/include/llvm/module.modulemap +++ b/llvm/include/llvm/module.modulemap @@ -4,6 +4,7 @@ module LLVM_Analysis { module * { export * } // This is intended for (repeated) textual inclusion. + textual header "Analysis/ScalarFuncs.def" textual header "Analysis/TargetLibraryInfo.def" textual header "Analysis/VecFuncs.def" } @@ -71,6 +72,7 @@ module LLVM_BinaryFormat { textual header "BinaryFormat/ELFRelocs/Hexagon.def" textual header "BinaryFormat/ELFRelocs/i386.def" textual header "BinaryFormat/ELFRelocs/Lanai.def" + textual header "BinaryFormat/ELFRelocs/LoongArch.def" textual header "BinaryFormat/ELFRelocs/M68k.def" textual header "BinaryFormat/ELFRelocs/Mips.def" textual header "BinaryFormat/ELFRelocs/MSP430.def" @@ -242,6 +244,7 @@ module LLVM_intrinsic_gen { export * } module IR_AbstractCallSite { header "IR/AbstractCallSite.h" export * } + module IR_ConstantFold { header "IR/ConstantFold.h" export * } module IR_ConstantFolder { header "IR/ConstantFolder.h" export * } module IR_GlobalVariable { header "IR/GlobalVariable.h" export * } module IR_NoFolder { header "IR/NoFolder.h" export * } @@ -253,6 +256,7 @@ module LLVM_intrinsic_gen { module IR_InstrTypes { header "IR/InstrTypes.h" export * } module IR_Instructions { header "IR/Instructions.h" export * } module IR_TypeFinder { header "IR/TypeFinder.h" export * } + module IR_VectorBuilder { header "IR/VectorBuilder.h" export * } // Intrinsics.h @@ -331,7 +335,6 @@ module LLVM_MC { module LLVM_MC_TableGen { requires cplusplus module MC_LaneBitmask { header "MC/LaneBitmask.h" export * } - module MC_FixedLenDisassembler { header "MC/MCFixedLenDisassembler.h" export * } module MC_InstrItineraries { header "MC/MCInstrItineraries.h" export * } module MC_Schedule { header "MC/MCSchedule.h" export * } module MC_SubtargetFeature { header "MC/SubtargetFeature.h" export * } @@ -357,6 +360,7 @@ module LLVM_ProfileData { textual header "ProfileData/InstrProfData.inc" textual header "ProfileData/MemProfData.inc" + textual header "ProfileData/MIBEntryDef.inc" } // FIXME: Mislayered? @@ -410,6 +414,7 @@ module LLVM_Utils { // These are intended for textual inclusion. textual header "Support/AArch64TargetParser.def" textual header "Support/ARMTargetParser.def" + textual header "Support/CSKYTargetParser.def" textual header "Support/RISCVTargetParser.def" textual header "Support/TargetOpcodes.def" textual header "Support/X86TargetParser.def" |