diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp | 217 |
1 files changed, 89 insertions, 128 deletions
diff --git a/contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp b/contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp index 8bd063116cf6..085554f18b2b 100644 --- a/contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp +++ b/contrib/llvm-project/llvm/lib/TargetParser/X86TargetParser.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/TargetParser/X86TargetParser.h" +#include "llvm/ADT/Bitset.h" #include "llvm/ADT/StringSwitch.h" #include <numeric> @@ -19,88 +20,7 @@ using namespace llvm::X86; namespace { -/// Container class for CPU features. -/// This is a constexpr reimplementation of a subset of std::bitset. It would be -/// nice to use std::bitset directly, but it doesn't support constant -/// initialization. -class FeatureBitset { - static constexpr unsigned NUM_FEATURE_WORDS = - (X86::CPU_FEATURE_MAX + 31) / 32; - - // This cannot be a std::array, operator[] is not constexpr until C++17. - uint32_t Bits[NUM_FEATURE_WORDS] = {}; - -public: - constexpr FeatureBitset() = default; - constexpr FeatureBitset(std::initializer_list<unsigned> Init) { - for (auto I : Init) - set(I); - } - - bool any() const { - return llvm::any_of(Bits, [](uint64_t V) { return V != 0; }); - } - - constexpr FeatureBitset &set(unsigned I) { - // GCC <6.2 crashes if this is written in a single statement. - uint32_t NewBits = Bits[I / 32] | (uint32_t(1) << (I % 32)); - Bits[I / 32] = NewBits; - return *this; - } - - constexpr bool operator[](unsigned I) const { - uint32_t Mask = uint32_t(1) << (I % 32); - return (Bits[I / 32] & Mask) != 0; - } - - constexpr FeatureBitset &operator&=(const FeatureBitset &RHS) { - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) { - // GCC <6.2 crashes if this is written in a single statement. - uint32_t NewBits = Bits[I] & RHS.Bits[I]; - Bits[I] = NewBits; - } - return *this; - } - - constexpr FeatureBitset &operator|=(const FeatureBitset &RHS) { - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) { - // GCC <6.2 crashes if this is written in a single statement. - uint32_t NewBits = Bits[I] | RHS.Bits[I]; - Bits[I] = NewBits; - } - return *this; - } - - // gcc 5.3 miscompiles this if we try to write this using operator&=. - constexpr FeatureBitset operator&(const FeatureBitset &RHS) const { - FeatureBitset Result; - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) - Result.Bits[I] = Bits[I] & RHS.Bits[I]; - return Result; - } - - // gcc 5.3 miscompiles this if we try to write this using operator&=. - constexpr FeatureBitset operator|(const FeatureBitset &RHS) const { - FeatureBitset Result; - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) - Result.Bits[I] = Bits[I] | RHS.Bits[I]; - return Result; - } - - constexpr FeatureBitset operator~() const { - FeatureBitset Result; - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) - Result.Bits[I] = ~Bits[I]; - return Result; - } - - constexpr bool operator!=(const FeatureBitset &RHS) const { - for (unsigned I = 0, E = std::size(Bits); I != E; ++I) - if (Bits[I] != RHS.Bits[I]) - return true; - return false; - } -}; +using FeatureBitset = Bitset<X86::CPU_FEATURE_MAX>; struct ProcInfo { StringLiteral Name; @@ -112,8 +32,15 @@ struct ProcInfo { }; struct FeatureInfo { - StringLiteral Name; + StringLiteral NameWithPlus; FeatureBitset ImpliedFeatures; + + StringRef getName(bool WithPlus = false) const { + assert(NameWithPlus[0] == '+' && "Expected string to start with '+'"); + if (WithPlus) + return NameWithPlus; + return NameWithPlus.drop_front(); + } }; } // end anonymous namespace @@ -145,7 +72,7 @@ constexpr FeatureBitset FeaturesX86_64_V2 = FeaturesX86_64 | FeatureSAHF | constexpr FeatureBitset FeaturesX86_64_V3 = FeaturesX86_64_V2 | FeatureAVX2 | FeatureBMI | FeatureBMI2 | FeatureF16C | FeatureFMA | FeatureLZCNT | FeatureMOVBE | FeatureXSAVE; -constexpr FeatureBitset FeaturesX86_64_V4 = FeaturesX86_64_V3 | +constexpr FeatureBitset FeaturesX86_64_V4 = FeaturesX86_64_V3 | FeatureEVEX512 | FeatureAVX512BW | FeatureAVX512CD | FeatureAVX512DQ | FeatureAVX512VL; @@ -169,8 +96,8 @@ constexpr FeatureBitset FeaturesBroadwell = // Intel Knights Landing and Knights Mill // Knights Landing has feature parity with Broadwell. constexpr FeatureBitset FeaturesKNL = - FeaturesBroadwell | FeatureAES | FeatureAVX512F | FeatureAVX512CD | - FeatureAVX512ER | FeatureAVX512PF | FeaturePREFETCHWT1; + FeaturesBroadwell | FeatureAES | FeatureAVX512F | FeatureEVEX512 | + FeatureAVX512CD | FeatureAVX512ER | FeatureAVX512PF | FeaturePREFETCHWT1; constexpr FeatureBitset FeaturesKNM = FeaturesKNL | FeatureAVX512VPOPCNTDQ; // Intel Skylake processors. @@ -180,9 +107,9 @@ constexpr FeatureBitset FeaturesSkylakeClient = // SkylakeServer inherits all SkylakeClient features except SGX. // FIXME: That doesn't match gcc. constexpr FeatureBitset FeaturesSkylakeServer = - (FeaturesSkylakeClient & ~FeatureSGX) | FeatureAVX512F | FeatureAVX512CD | - FeatureAVX512DQ | FeatureAVX512BW | FeatureAVX512VL | FeatureCLWB | - FeaturePKU; + (FeaturesSkylakeClient & ~FeatureSGX) | FeatureAVX512F | FeatureEVEX512 | + FeatureAVX512CD | FeatureAVX512DQ | FeatureAVX512BW | FeatureAVX512VL | + FeatureCLWB | FeaturePKU; constexpr FeatureBitset FeaturesCascadeLake = FeaturesSkylakeServer | FeatureAVX512VNNI; constexpr FeatureBitset FeaturesCooperLake = @@ -190,9 +117,9 @@ constexpr FeatureBitset FeaturesCooperLake = // Intel 10nm processors. constexpr FeatureBitset FeaturesCannonlake = - FeaturesSkylakeClient | FeatureAVX512F | FeatureAVX512CD | FeatureAVX512DQ | - FeatureAVX512BW | FeatureAVX512VL | FeatureAVX512IFMA | FeatureAVX512VBMI | - FeaturePKU | FeatureSHA; + FeaturesSkylakeClient | FeatureAVX512F | FeatureEVEX512 | FeatureAVX512CD | + FeatureAVX512DQ | FeatureAVX512BW | FeatureAVX512VL | FeatureAVX512IFMA | + FeatureAVX512VBMI | FeaturePKU | FeatureSHA; constexpr FeatureBitset FeaturesICLClient = FeaturesCannonlake | FeatureAVX512BITALG | FeatureAVX512VBMI2 | FeatureAVX512VNNI | FeatureAVX512VPOPCNTDQ | FeatureGFNI | FeatureRDPID | @@ -237,6 +164,12 @@ constexpr FeatureBitset FeaturesSierraforest = FeatureENQCMD | FeatureAVXNECONVERT | FeatureAVXVNNIINT8; constexpr FeatureBitset FeaturesGrandridge = FeaturesSierraforest | FeatureRAOINT; +constexpr FeatureBitset FeaturesArrowlakeS = FeaturesSierraforest | + FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 | FeatureSM4; +constexpr FeatureBitset FeaturesPantherlake = + FeaturesArrowlakeS | FeaturePREFETCHI; +constexpr FeatureBitset FeaturesClearwaterforest = + FeaturesArrowlakeS | FeatureUSERMSR | FeaturePREFETCHI; // Geode Processor. constexpr FeatureBitset FeaturesGeode = @@ -301,11 +234,11 @@ static constexpr FeatureBitset FeaturesZNVER3 = FeaturesZNVER2 | FeatureINVPCID | FeaturePKU | FeatureVAES | FeatureVPCLMULQDQ; static constexpr FeatureBitset FeaturesZNVER4 = - FeaturesZNVER3 | FeatureAVX512F | FeatureAVX512CD | FeatureAVX512DQ | - FeatureAVX512BW | FeatureAVX512VL | FeatureAVX512IFMA | FeatureAVX512VBMI | - FeatureAVX512VBMI2 | FeatureAVX512VNNI | FeatureAVX512BITALG | - FeatureAVX512VPOPCNTDQ | FeatureAVX512BF16 | FeatureGFNI | - FeatureSHSTK; + FeaturesZNVER3 | FeatureAVX512F | FeatureEVEX512 | FeatureAVX512CD | + FeatureAVX512DQ | FeatureAVX512BW | FeatureAVX512VL | FeatureAVX512IFMA | + FeatureAVX512VBMI | FeatureAVX512VBMI2 | FeatureAVX512VNNI | + FeatureAVX512BITALG | FeatureAVX512VPOPCNTDQ | FeatureAVX512BF16 | + FeatureGFNI | FeatureSHSTK; // D151696 tranplanted Mangling and OnlyForCPUDispatchSpecific from // X86TargetParser.def to here. They are assigned by following ways: @@ -315,6 +248,7 @@ static constexpr FeatureBitset FeaturesZNVER4 = // listed here before, which means it doesn't support -march, -mtune and so on. // FIXME: Remove OnlyForCPUDispatchSpecific after all CPUs here support both // cpu_dispatch/specific() feature and -march, -mtune, and so on. +// clang-format off constexpr ProcInfo Processors[] = { // Empty processor. Include X87 and CMPXCHG8 for backwards compatibility. { {""}, CK_None, ~0U, FeatureX87 | FeatureCMPXCHG8B, '\0', false }, @@ -422,6 +356,16 @@ constexpr ProcInfo Processors[] = { { {"raptorlake"}, CK_Raptorlake, FEATURE_AVX2, FeaturesAlderlake, 'p', false }, // Meteorlake microarchitecture based processors. { {"meteorlake"}, CK_Meteorlake, FEATURE_AVX2, FeaturesAlderlake, 'p', false }, + // Arrowlake microarchitecture based processors. + { {"arrowlake"}, CK_Arrowlake, FEATURE_AVX2, FeaturesSierraforest, 'p', false }, + { {"arrowlake-s"}, CK_ArrowlakeS, FEATURE_AVX2, FeaturesArrowlakeS, '\0', false }, + { {"arrowlake_s"}, CK_ArrowlakeS, FEATURE_AVX2, FeaturesArrowlakeS, 'p', true }, + // Lunarlake microarchitecture based processors. + { {"lunarlake"}, CK_Lunarlake, FEATURE_AVX2, FeaturesArrowlakeS, 'p', false }, + // Gracemont microarchitecture based processors. + { {"gracemont"}, CK_Gracemont, FEATURE_AVX2, FeaturesAlderlake, 'p', false }, + // Pantherlake microarchitecture based processors. + { {"pantherlake"}, CK_Lunarlake, FEATURE_AVX2, FeaturesPantherlake, 'p', false }, // Sierraforest microarchitecture based processors. { {"sierraforest"}, CK_Sierraforest, FEATURE_AVX2, FeaturesSierraforest, 'p', false }, // Grandridge microarchitecture based processors. @@ -433,6 +377,8 @@ constexpr ProcInfo Processors[] = { { {"graniterapids_d"}, CK_GraniterapidsD, FEATURE_AVX512BF16, FeaturesGraniteRapids | FeatureAMX_COMPLEX, 'n', true }, // Emerald Rapids microarchitecture based processors. { {"emeraldrapids"}, CK_Emeraldrapids, FEATURE_AVX512BF16, FeaturesSapphireRapids, 'n', false }, + // Clearwaterforest microarchitecture based processors. + { {"clearwaterforest"}, CK_Lunarlake, FEATURE_AVX2, FeaturesClearwaterforest, 'p', false }, // Knights Landing processor. { {"knl"}, CK_KNL, FEATURE_AVX512F, FeaturesKNL, 'Z', false }, { {"mic_avx512"}, CK_KNL, FEATURE_AVX512F, FeaturesKNL, 'Z', true }, @@ -474,13 +420,14 @@ constexpr ProcInfo Processors[] = { { {"znver3"}, CK_ZNVER3, FEATURE_AVX2, FeaturesZNVER3, '\0', false }, { {"znver4"}, CK_ZNVER4, FEATURE_AVX512VBMI2, FeaturesZNVER4, '\0', false }, // Generic 64-bit processor. - { {"x86-64"}, CK_x86_64, ~0U, FeaturesX86_64, '\0', false }, - { {"x86-64-v2"}, CK_x86_64_v2, ~0U, FeaturesX86_64_V2, '\0', false }, - { {"x86-64-v3"}, CK_x86_64_v3, ~0U, FeaturesX86_64_V3, '\0', false }, - { {"x86-64-v4"}, CK_x86_64_v4, ~0U, FeaturesX86_64_V4, '\0', false }, + { {"x86-64"}, CK_x86_64, FEATURE_SSE2 , FeaturesX86_64, '\0', false }, + { {"x86-64-v2"}, CK_x86_64_v2, FEATURE_SSE4_2 , FeaturesX86_64_V2, '\0', false }, + { {"x86-64-v3"}, CK_x86_64_v3, FEATURE_AVX2, FeaturesX86_64_V3, '\0', false }, + { {"x86-64-v4"}, CK_x86_64_v4, FEATURE_AVX512VL, FeaturesX86_64_V4, '\0', false }, // Geode processors. { {"geode"}, CK_Geode, ~0U, FeaturesGeode, '\0', false }, }; +// clang-format on constexpr const char *NoTuneList[] = {"x86-64-v2", "x86-64-v3", "x86-64-v4"}; @@ -570,6 +517,7 @@ constexpr FeatureBitset ImpliedFeaturesSHSTK = {}; constexpr FeatureBitset ImpliedFeaturesTBM = {}; constexpr FeatureBitset ImpliedFeaturesTSXLDTRK = {}; constexpr FeatureBitset ImpliedFeaturesUINTR = {}; +constexpr FeatureBitset ImpliedFeaturesUSERMSR = {}; constexpr FeatureBitset ImpliedFeaturesWAITPKG = {}; constexpr FeatureBitset ImpliedFeaturesWBNOINVD = {}; constexpr FeatureBitset ImpliedFeaturesVZEROUPPER = {}; @@ -603,6 +551,7 @@ constexpr FeatureBitset ImpliedFeaturesSSE4_1 = FeatureSSSE3; constexpr FeatureBitset ImpliedFeaturesSSE4_2 = FeatureSSE4_1; constexpr FeatureBitset ImpliedFeaturesAVX = FeatureSSE4_2; constexpr FeatureBitset ImpliedFeaturesAVX2 = FeatureAVX; +constexpr FeatureBitset ImpliedFeaturesEVEX512 = {}; constexpr FeatureBitset ImpliedFeaturesAVX512F = FeatureAVX2 | FeatureF16C | FeatureFMA; @@ -613,10 +562,10 @@ constexpr FeatureBitset ImpliedFeaturesFMA = FeatureAVX; constexpr FeatureBitset ImpliedFeaturesGFNI = FeatureSSE2; constexpr FeatureBitset ImpliedFeaturesPCLMUL = FeatureSSE2; constexpr FeatureBitset ImpliedFeaturesSHA = FeatureSSE2; -constexpr FeatureBitset ImpliedFeaturesVAES = FeatureAES | FeatureAVX; +constexpr FeatureBitset ImpliedFeaturesVAES = FeatureAES | FeatureAVX2; constexpr FeatureBitset ImpliedFeaturesVPCLMULQDQ = FeatureAVX | FeaturePCLMUL; constexpr FeatureBitset ImpliedFeaturesSM3 = FeatureAVX; -constexpr FeatureBitset ImpliedFeaturesSM4 = FeatureAVX; +constexpr FeatureBitset ImpliedFeaturesSM4 = FeatureAVX2; // AVX512 features. constexpr FeatureBitset ImpliedFeaturesAVX512CD = FeatureAVX512F; @@ -660,7 +609,7 @@ constexpr FeatureBitset ImpliedFeaturesAVXVNNIINT16 = FeatureAVX2; constexpr FeatureBitset ImpliedFeaturesAVXVNNIINT8 = FeatureAVX2; constexpr FeatureBitset ImpliedFeaturesAVXIFMA = FeatureAVX2; constexpr FeatureBitset ImpliedFeaturesAVXNECONVERT = FeatureAVX2; -constexpr FeatureBitset ImpliedFeaturesSHA512 = FeatureAVX; +constexpr FeatureBitset ImpliedFeaturesSHA512 = FeatureAVX2; constexpr FeatureBitset ImpliedFeaturesAVX512FP16 = FeatureAVX512BW | FeatureAVX512DQ | FeatureAVX512VL; // Key Locker Features @@ -670,19 +619,31 @@ constexpr FeatureBitset ImpliedFeaturesWIDEKL = FeatureKL; // AVXVNNI Features constexpr FeatureBitset ImpliedFeaturesAVXVNNI = FeatureAVX2; -constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = { -#define X86_FEATURE(ENUM, STR) {{STR}, ImpliedFeatures##ENUM}, -#include "llvm/TargetParser/X86TargetParser.def" -}; +// AVX10 Features +constexpr FeatureBitset ImpliedFeaturesAVX10_1 = + FeatureAVX512CD | FeatureAVX512VBMI | FeatureAVX512IFMA | + FeatureAVX512VNNI | FeatureAVX512BF16 | FeatureAVX512VPOPCNTDQ | + FeatureAVX512VBMI2 | FeatureAVX512BITALG | FeatureVAES | FeatureVPCLMULQDQ | + FeatureAVX512FP16; +constexpr FeatureBitset ImpliedFeaturesAVX10_1_512 = + FeatureAVX10_1 | FeatureEVEX512; + +// APX Features +constexpr FeatureBitset ImpliedFeaturesEGPR = {}; +constexpr FeatureBitset ImpliedFeaturesPush2Pop2 = {}; +constexpr FeatureBitset ImpliedFeaturesPPX = {}; +constexpr FeatureBitset ImpliedFeaturesNDD = {}; +constexpr FeatureBitset ImpliedFeaturesCCMP = {}; +constexpr FeatureBitset ImpliedFeaturesCF = {}; -constexpr FeatureInfo FeatureInfos_WithPLUS[X86::CPU_FEATURE_MAX] = { +constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = { #define X86_FEATURE(ENUM, STR) {{"+" STR}, ImpliedFeatures##ENUM}, #include "llvm/TargetParser/X86TargetParser.def" }; void llvm::X86::getFeaturesForCPU(StringRef CPU, SmallVectorImpl<StringRef> &EnabledFeatures, - bool IfNeedPlus) { + bool NeedPlus) { auto I = llvm::find_if(Processors, [&](const ProcInfo &P) { return P.Name == CPU; }); assert(I != std::end(Processors) && "Processor not found!"); @@ -695,11 +656,8 @@ void llvm::X86::getFeaturesForCPU(StringRef CPU, // Add the string version of all set bits. for (unsigned i = 0; i != CPU_FEATURE_MAX; ++i) - if (Bits[i] && !FeatureInfos[i].Name.empty() && - !FeatureInfos_WithPLUS[i].Name.empty()){ - EnabledFeatures.push_back(IfNeedPlus ? FeatureInfos_WithPLUS[i].Name - : FeatureInfos[i].Name); - } + if (Bits[i] && !FeatureInfos[i].getName(NeedPlus).empty()) + EnabledFeatures.push_back(FeatureInfos[i].getName(NeedPlus)); } // For each feature that is (transitively) implied by this feature, set it. @@ -736,8 +694,9 @@ static void getImpliedDisabledFeatures(FeatureBitset &Bits, unsigned Value) { void llvm::X86::updateImpliedFeatures( StringRef Feature, bool Enabled, StringMap<bool> &Features) { - auto I = llvm::find_if( - FeatureInfos, [&](const FeatureInfo &FI) { return FI.Name == Feature; }); + auto I = llvm::find_if(FeatureInfos, [&](const FeatureInfo &FI) { + return FI.getName() == Feature; + }); if (I == std::end(FeatureInfos)) { // FIXME: This shouldn't happen, but may not have all features in the table // yet. @@ -753,8 +712,8 @@ void llvm::X86::updateImpliedFeatures( // Update the map entry for all implied features. for (unsigned i = 0; i != CPU_FEATURE_MAX; ++i) - if (ImpliedBits[i] && !FeatureInfos[i].Name.empty()) - Features[FeatureInfos[i].Name] = Enabled; + if (ImpliedBits[i] && !FeatureInfos[i].getName().empty()) + Features[FeatureInfos[i].getName()] = Enabled; } char llvm::X86::getCPUDispatchMangling(StringRef CPU) { @@ -771,18 +730,22 @@ bool llvm::X86::validateCPUSpecificCPUDispatch(StringRef Name) { return I != std::end(Processors); } -uint64_t llvm::X86::getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs) { +std::array<uint32_t, 4> +llvm::X86::getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs) { // Processor features and mapping to processor feature value. - uint64_t FeaturesMask = 0; - for (const StringRef &FeatureStr : FeatureStrs) { + std::array<uint32_t, 4> FeatureMask{}; + for (StringRef FeatureStr : FeatureStrs) { unsigned Feature = StringSwitch<unsigned>(FeatureStr) #define X86_FEATURE_COMPAT(ENUM, STR, PRIORITY) \ .Case(STR, llvm::X86::FEATURE_##ENUM) +#define X86_MICROARCH_LEVEL(ENUM, STR, PRIORITY) \ + .Case(STR, llvm::X86::FEATURE_##ENUM) #include "llvm/TargetParser/X86TargetParser.def" ; - FeaturesMask |= (1ULL << Feature); + assert(Feature / 32 < FeatureMask.size()); + FeatureMask[Feature / 32] |= 1U << (Feature % 32); } - return FeaturesMask; + return FeatureMask; } unsigned llvm::X86::getFeaturePriority(ProcessorFeatures Feat) { @@ -793,13 +756,11 @@ unsigned llvm::X86::getFeaturePriority(ProcessorFeatures Feat) { #define X86_FEATURE_COMPAT(ENUM, STR, PRIORITY) PRIORITY, unsigned Priorities[] = { #include "llvm/TargetParser/X86TargetParser.def" - std::numeric_limits<unsigned>::max() // Need to consume last comma. }; - std::array<unsigned, std::size(Priorities) - 1> HelperList; + std::array<unsigned, std::size(Priorities)> HelperList; std::iota(HelperList.begin(), HelperList.end(), 0); assert(std::is_permutation(HelperList.begin(), HelperList.end(), - std::begin(Priorities), - std::prev(std::end(Priorities))) && + std::begin(Priorities), std::end(Priorities)) && "Priorities don't form consecutive range!"); #endif |