diff options
Diffstat (limited to 'llvm')
28 files changed, 457 insertions, 147 deletions
diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h index caece0e6fc19..ec7f5691dda4 100644 --- a/llvm/include/llvm/Debuginfod/Debuginfod.h +++ b/llvm/include/llvm/Debuginfod/Debuginfod.h @@ -38,9 +38,13 @@ namespace llvm { +/// Returns false if a debuginfod lookup can be determined to have no chance of +/// succeeding. +bool canUseDebuginfod(); + /// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS /// environment variable. -Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls(); +SmallVector<StringRef> getDefaultDebuginfodUrls(); /// Finds a default local file caching directory for the debuginfod client, /// first checking DEBUGINFOD_CACHE_PATH. diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index b1f85563195f..92a198befbe4 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1391,6 +1391,16 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". +class AdvSIMD_SVE_2SVBoolArg_Intrinsic + : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], + [llvm_nxv16i1_ty], + [IntrNoMem]>; + +class AdvSIMD_SVE_3SVBoolArg_Intrinsic + : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], + [llvm_nxv16i1_ty, llvm_nxv16i1_ty], + [IntrNoMem]>; + class AdvSIMD_SVE_Reduce_Intrinsic : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, @@ -1836,22 +1846,43 @@ def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; +def int_aarch64_sve_rev_b16 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; +def int_aarch64_sve_rev_b32 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; +def int_aarch64_sve_rev_b64 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_trn1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_trn1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_trn1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_trn2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_trn2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_trn2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_uzp1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_uzp1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_uzp1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_uzp2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_uzp2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_uzp2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_zip1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_zip1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_zip1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_zip2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_zip2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; +def int_aarch64_sve_zip2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 4d48308d5509..bdb772862468 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -21,6 +21,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" @@ -42,6 +43,10 @@ namespace llvm { class IndexedInstrProfReader; +namespace object { +class BuildIDFetcher; +} // namespace object + namespace coverage { class CoverageMappingReader; @@ -579,6 +584,13 @@ class CoverageMapping { ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders, IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage); + // Load coverage records from file. + static Error + loadFromFile(StringRef Filename, StringRef Arch, StringRef CompilationDir, + IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage, + bool &DataFound, + SmallVectorImpl<object::BuildID> *FoundBinaryIDs = nullptr); + /// Add a function record corresponding to \p Record. Error loadFunctionRecord(const CoverageMappingRecord &Record, IndexedInstrProfReader &ProfileReader); @@ -604,8 +616,8 @@ public: /// Ignores non-instrumented object files unless all are not instrumented. static Expected<std::unique_ptr<CoverageMapping>> load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename, - ArrayRef<StringRef> Arches = std::nullopt, - StringRef CompilationDir = ""); + ArrayRef<StringRef> Arches = std::nullopt, StringRef CompilationDir = "", + const object::BuildIDFetcher *BIDFetcher = nullptr); /// The number of functions that couldn't have their profiles mapped. /// diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 39c0045369be..326c1b0d3338 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -205,7 +205,8 @@ public: static Expected<std::vector<std::unique_ptr<BinaryCoverageReader>>> create(MemoryBufferRef ObjectBuffer, StringRef Arch, SmallVectorImpl<std::unique_ptr<MemoryBuffer>> &ObjectFileBuffers, - StringRef CompilationDir = ""); + StringRef CompilationDir = "", + SmallVectorImpl<object::BuildIDRef> *BinaryIDs = nullptr); static Expected<std::unique_ptr<BinaryCoverageReader>> createCoverageReaderFromBuffer(StringRef Coverage, diff --git a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h index bf08336663b6..73aee47bfef5 100644 --- a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h +++ b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h @@ -37,13 +37,25 @@ KernelSet getDeviceKernels(Module &M); /// OpenMP optimizations pass. class OpenMPOptPass : public PassInfoMixin<OpenMPOptPass> { public: + OpenMPOptPass() : LTOPhase(ThinOrFullLTOPhase::None) {} + OpenMPOptPass(ThinOrFullLTOPhase LTOPhase) : LTOPhase(LTOPhase) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + const ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None; }; class OpenMPOptCGSCCPass : public PassInfoMixin<OpenMPOptCGSCCPass> { public: + OpenMPOptCGSCCPass() : LTOPhase(ThinOrFullLTOPhase::None) {} + OpenMPOptCGSCCPass(ThinOrFullLTOPhase LTOPhase) : LTOPhase(LTOPhase) {} + PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR); + +private: + const ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None; }; } // end namespace llvm diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 8c126d20fc9a..0b1e32c87fc3 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2235,6 +2235,8 @@ bool AsmPrinter::doFinalization(Module &M) { SmallVector<const GlobalAlias *, 16> AliasStack; SmallPtrSet<const GlobalAlias *, 16> AliasVisited; for (const auto &Alias : M.aliases()) { + if (Alias.hasAvailableExternallyLinkage()) + continue; for (const GlobalAlias *Cur = &Alias; Cur; Cur = dyn_cast<GlobalAlias>(Cur->getAliasee())) { if (!AliasVisited.insert(Cur).second) diff --git a/llvm/lib/CodeGen/IfConversion.cpp b/llvm/lib/CodeGen/IfConversion.cpp index 105ab908d3fa..936de316e575 100644 --- a/llvm/lib/CodeGen/IfConversion.cpp +++ b/llvm/lib/CodeGen/IfConversion.cpp @@ -2244,6 +2244,15 @@ void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) { assert(!FromMBB.hasAddressTaken() && "Removing a BB whose address is taken!"); + // If we're about to splice an INLINEASM_BR from FromBBI, we need to update + // ToBBI's successor list accordingly. + if (FromMBB.mayHaveInlineAsmBr()) + for (MachineInstr &MI : FromMBB) + if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) + for (MachineOperand &MO : MI.operands()) + if (MO.isMBB() && !ToBBI.BB->isSuccessor(MO.getMBB())) + ToBBI.BB->addSuccessor(MO.getMBB(), BranchProbability::getZero()); + // In case FromMBB contains terminators (e.g. return instruction), // first move the non-terminator instructions, then the terminators. MachineBasicBlock::iterator FromTI = FromMBB.getFirstTerminator(); diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp index 026f118bbf5b..2b0710b536ba 100644 --- a/llvm/lib/Debuginfod/Debuginfod.cpp +++ b/llvm/lib/Debuginfod/Debuginfod.cpp @@ -55,7 +55,11 @@ static std::string buildIDToString(BuildIDRef ID) { return llvm::toHex(ID, /*LowerCase=*/true); } -Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() { +bool canUseDebuginfod() { + return HTTPClient::isAvailable() && !getDefaultDebuginfodUrls().empty(); +} + +SmallVector<StringRef> getDefaultDebuginfodUrls() { const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); if (DebuginfodUrlsEnv == nullptr) return SmallVector<StringRef>(); @@ -126,13 +130,8 @@ Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey, return CacheDirOrErr.takeError(); CacheDir = *CacheDirOrErr; - Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr = - getDefaultDebuginfodUrls(); - if (!DebuginfodUrlsOrErr) - return DebuginfodUrlsOrErr.takeError(); - SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr; return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir, - DebuginfodUrls, + getDefaultDebuginfodUrls(), getDefaultDebuginfodTimeout()); } @@ -159,7 +158,8 @@ public: Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) { if (!FileStream) { - if (Client.responseCode() != 200) + unsigned Code = Client.responseCode(); + if (Code && Code != 200) return Error::success(); Expected<std::unique_ptr<CachedFileStream>> FileStreamOrError = CreateStream(); @@ -259,7 +259,8 @@ Expected<std::string> getCachedOrDownloadArtifact( if (Err) return std::move(Err); - if (Client.responseCode() != 200) + unsigned Code = Client.responseCode(); + if (Code && Code != 200) continue; // Return the path to the artifact on disk. diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 95380d912392..3368d3276cb3 100644 --- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -125,6 +125,10 @@ void IRMaterializationUnit::discard(const JITDylib &JD, assert(!I->second->isDeclaration() && "Discard should only apply to definitions"); I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + // According to the IR verifier, "Declaration[s] may not be in a Comdat!" + // Remove it, if this is a GlobalObject. + if (auto *GO = dyn_cast<GlobalObject>(I->second)) + GO->setComdat(nullptr); SymbolToDefinition.erase(I); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 83e42bc184ff..2d62d31cf6b4 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -730,9 +730,6 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { GV.getName() == "llvm.global_dtors")) { Check(!GV.hasInitializer() || GV.hasAppendingLinkage(), "invalid linkage for intrinsic global variable", &GV); - Check(GV.materialized_use_empty(), - "invalid uses of intrinsic global variable", &GV); - // Don't worry about emitting an error for it not being an array, // visitGlobalValue will complain on appending non-array. if (ArrayType *ATy = dyn_cast<ArrayType>(GV.getValueType())) { @@ -759,9 +756,6 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { GV.getName() == "llvm.compiler.used")) { Check(!GV.hasInitializer() || GV.hasAppendingLinkage(), "invalid linkage for intrinsic global variable", &GV); - Check(GV.materialized_use_empty(), - "invalid uses of intrinsic global variable", &GV); - Type *GVType = GV.getValueType(); if (ArrayType *ATy = dyn_cast<ArrayType>(GVType)) { PointerType *PTy = dyn_cast<PointerType>(ATy->getElementType()); diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 0762c535f7f5..eed29c25714b 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1604,7 +1604,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, } // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present. - MPM.addPass(OpenMPOptPass()); + MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink)); // Remove unused virtual tables to improve the quality of code generated by // whole-program devirtualization and bitset lowering. @@ -1712,6 +1712,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // Optimize globals again after we ran the inliner. MPM.addPass(GlobalOptPass()); + // Run the OpenMPOpt pass again after global optimizations. + MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink)); + // Garbage collect dead functions. MPM.addPass(GlobalDCEPass()); @@ -1808,7 +1811,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, addVectorPasses(Level, MainFPM, /* IsFullLTO */ true); // Run the OpenMPOpt CGSCC pass again late. - MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(OpenMPOptCGSCCPass())); + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink))); invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass()); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index ad44d86ea1a7..10af4160c545 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -44,6 +44,7 @@ MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("attributor", AttributorPass()) MODULE_PASS("annotation2metadata", Annotation2MetadataPass()) MODULE_PASS("openmp-opt", OpenMPOptPass()) +MODULE_PASS("openmp-opt-postlink", OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink)) MODULE_PASS("called-value-propagation", CalledValuePropagationPass()) MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass()) MODULE_PASS("cg-profile", CGProfilePass()) diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index 6113f78aeb4e..ce71eebd4fd3 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/Coverage/CoverageMappingReader.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/Debug.h" @@ -342,10 +343,49 @@ static Error handleMaybeNoDataFoundError(Error E) { }); } +Error CoverageMapping::loadFromFile( + StringRef Filename, StringRef Arch, StringRef CompilationDir, + IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage, + bool &DataFound, SmallVectorImpl<object::BuildID> *FoundBinaryIDs) { + auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN( + Filename, /*IsText=*/false, /*RequiresNullTerminator=*/false); + if (std::error_code EC = CovMappingBufOrErr.getError()) + return createFileError(Filename, errorCodeToError(EC)); + MemoryBufferRef CovMappingBufRef = + CovMappingBufOrErr.get()->getMemBufferRef(); + SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers; + + SmallVector<object::BuildIDRef> BinaryIDs; + auto CoverageReadersOrErr = BinaryCoverageReader::create( + CovMappingBufRef, Arch, Buffers, CompilationDir, + FoundBinaryIDs ? &BinaryIDs : nullptr); + if (Error E = CoverageReadersOrErr.takeError()) { + E = handleMaybeNoDataFoundError(std::move(E)); + if (E) + return createFileError(Filename, std::move(E)); + return E; + } + + SmallVector<std::unique_ptr<CoverageMappingReader>, 4> Readers; + for (auto &Reader : CoverageReadersOrErr.get()) + Readers.push_back(std::move(Reader)); + if (FoundBinaryIDs && !Readers.empty()) { + llvm::append_range(*FoundBinaryIDs, + llvm::map_range(BinaryIDs, [](object::BuildIDRef BID) { + return object::BuildID(BID); + })); + } + DataFound |= !Readers.empty(); + if (Error E = loadFromReaders(Readers, ProfileReader, Coverage)) + return createFileError(Filename, std::move(E)); + return Error::success(); +} + Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename, ArrayRef<StringRef> Arches, - StringRef CompilationDir) { + StringRef CompilationDir, + const object::BuildIDFetcher *BIDFetcher) { auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename); if (Error E = ProfileReaderOrErr.takeError()) return createFileError(ProfileFilename, std::move(E)); @@ -353,35 +393,53 @@ CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames, auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping()); bool DataFound = false; + auto GetArch = [&](size_t Idx) { + if (Arches.empty()) + return StringRef(); + if (Arches.size() == 1) + return Arches.front(); + return Arches[Idx]; + }; + + SmallVector<object::BuildID> FoundBinaryIDs; for (const auto &File : llvm::enumerate(ObjectFilenames)) { - auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN( - File.value(), /*IsText=*/false, /*RequiresNullTerminator=*/false); - if (std::error_code EC = CovMappingBufOrErr.getError()) - return createFileError(File.value(), errorCodeToError(EC)); - StringRef Arch = Arches.empty() ? StringRef() : Arches[File.index()]; - MemoryBufferRef CovMappingBufRef = - CovMappingBufOrErr.get()->getMemBufferRef(); - SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers; - auto CoverageReadersOrErr = BinaryCoverageReader::create( - CovMappingBufRef, Arch, Buffers, CompilationDir); - if (Error E = CoverageReadersOrErr.takeError()) { - E = handleMaybeNoDataFoundError(std::move(E)); - if (E) - return createFileError(File.value(), std::move(E)); - // E == success (originally a no_data_found error). - continue; + if (Error E = + loadFromFile(File.value(), GetArch(File.index()), CompilationDir, + *ProfileReader, *Coverage, DataFound, &FoundBinaryIDs)) + return std::move(E); + } + + if (BIDFetcher) { + std::vector<object::BuildID> ProfileBinaryIDs; + if (Error E = ProfileReader->readBinaryIds(ProfileBinaryIDs)) + return createFileError(ProfileFilename, std::move(E)); + + SmallVector<object::BuildIDRef> BinaryIDsToFetch; + if (!ProfileBinaryIDs.empty()) { + const auto &Compare = [](object::BuildIDRef A, object::BuildIDRef B) { + return std::lexicographical_compare(A.begin(), A.end(), B.begin(), + B.end()); + }; + llvm::sort(FoundBinaryIDs, Compare); + std::set_difference( + ProfileBinaryIDs.begin(), ProfileBinaryIDs.end(), + FoundBinaryIDs.begin(), FoundBinaryIDs.end(), + std::inserter(BinaryIDsToFetch, BinaryIDsToFetch.end()), Compare); } - SmallVector<std::unique_ptr<CoverageMappingReader>, 4> Readers; - for (auto &Reader : CoverageReadersOrErr.get()) - Readers.push_back(std::move(Reader)); - DataFound |= !Readers.empty(); - if (Error E = loadFromReaders(Readers, *ProfileReader, *Coverage)) - return createFileError(File.value(), std::move(E)); + for (object::BuildIDRef BinaryID : BinaryIDsToFetch) { + std::optional<std::string> PathOpt = BIDFetcher->fetch(BinaryID); + if (!PathOpt) + continue; + std::string Path = std::move(*PathOpt); + StringRef Arch = Arches.size() == 1 ? Arches.front() : StringRef(); + if (Error E = loadFromFile(Path, Arch, CompilationDir, *ProfileReader, + *Coverage, DataFound)) + return std::move(E); + } } - // If no readers were created, either no objects were provided or none of them - // had coverage data. Return an error in the latter case. - if (!DataFound && !ObjectFilenames.empty()) + + if (!DataFound) return createFileError( join(ObjectFilenames.begin(), ObjectFilenames.end(), ", "), make_error<CoverageMapError>(coveragemap_error::no_data_found)); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 41962ab24ff9..d313864e2ddb 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -954,7 +954,8 @@ static Expected<std::vector<SectionRef>> lookupSections(ObjectFile &OF, static Expected<std::unique_ptr<BinaryCoverageReader>> loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch, - StringRef CompilationDir = "") { + StringRef CompilationDir = "", + std::optional<object::BuildIDRef> *BinaryID = nullptr) { std::unique_ptr<ObjectFile> OF; if (auto *Universal = dyn_cast<MachOUniversalBinary>(Bin.get())) { // If we have a universal binary, try to look up the object for the @@ -1052,6 +1053,9 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch, FuncRecords = std::move(WritableBuffer); } + if (BinaryID) + *BinaryID = getBuildID(OF.get()); + return BinaryCoverageReader::createCoverageReaderFromBuffer( CoverageMapping, std::move(FuncRecords), std::move(ProfileNames), BytesInAddress, Endian, CompilationDir); @@ -1074,7 +1078,7 @@ Expected<std::vector<std::unique_ptr<BinaryCoverageReader>>> BinaryCoverageReader::create( MemoryBufferRef ObjectBuffer, StringRef Arch, SmallVectorImpl<std::unique_ptr<MemoryBuffer>> &ObjectFileBuffers, - StringRef CompilationDir) { + StringRef CompilationDir, SmallVectorImpl<object::BuildIDRef> *BinaryIDs) { std::vector<std::unique_ptr<BinaryCoverageReader>> Readers; if (ObjectBuffer.getBuffer().startswith(TestingFormatMagic)) { @@ -1114,7 +1118,7 @@ BinaryCoverageReader::create( return BinaryCoverageReader::create( ArchiveOrErr.get()->getMemoryBufferRef(), Arch, ObjectFileBuffers, - CompilationDir); + CompilationDir, BinaryIDs); } } @@ -1127,7 +1131,8 @@ BinaryCoverageReader::create( return ChildBufOrErr.takeError(); auto ChildReadersOrErr = BinaryCoverageReader::create( - ChildBufOrErr.get(), Arch, ObjectFileBuffers, CompilationDir); + ChildBufOrErr.get(), Arch, ObjectFileBuffers, CompilationDir, + BinaryIDs); if (!ChildReadersOrErr) return ChildReadersOrErr.takeError(); for (auto &Reader : ChildReadersOrErr.get()) @@ -1146,10 +1151,14 @@ BinaryCoverageReader::create( return std::move(Readers); } - auto ReaderOrErr = loadBinaryFormat(std::move(Bin), Arch, CompilationDir); + std::optional<object::BuildIDRef> BinaryID; + auto ReaderOrErr = loadBinaryFormat(std::move(Bin), Arch, CompilationDir, + BinaryIDs ? &BinaryID : nullptr); if (!ReaderOrErr) return ReaderOrErr.takeError(); Readers.push_back(std::move(ReaderOrErr.get())); + if (BinaryID) + BinaryIDs->push_back(*BinaryID); return std::move(Readers); } diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp index 23ed9d813548..c256d256be4f 100644 --- a/llvm/lib/Support/Parallel.cpp +++ b/llvm/lib/Support/Parallel.cpp @@ -214,8 +214,12 @@ void llvm::parallelFor(size_t Begin, size_t End, Fn(I); }); } - for (; Begin != End; ++Begin) - Fn(Begin); + if (Begin != End) { + TG.spawn([=, &Fn] { + for (size_t I = Begin; I != End; ++I) + Fn(I); + }); + } return; } #endif diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp index 92b15f14c62f..8943c4478c7f 100644 --- a/llvm/lib/Support/raw_ostream.cpp +++ b/llvm/lib/Support/raw_ostream.cpp @@ -802,8 +802,6 @@ uint64_t raw_fd_ostream::seek(uint64_t off) { flush(); #ifdef _WIN32 pos = ::_lseeki64(FD, off, SEEK_SET); -#elif defined(HAVE_LSEEK64) - pos = ::lseek64(FD, off, SEEK_SET); #else pos = ::lseek(FD, off, SEEK_SET); #endif diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 6a42c4ff31dc..12dc30a2818b 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -839,7 +839,7 @@ let Predicates = [HasSVEorSME] in { defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", AArch64revh_mt>; defm REVW_ZPmZ : sve_int_perm_rev_revw<"revw", AArch64revw_mt>; - defm REV_PP : sve_int_perm_reverse_p<"rev", vector_reverse>; + defm REV_PP : sve_int_perm_reverse_p<"rev", vector_reverse, int_aarch64_sve_rev_b16, int_aarch64_sve_rev_b32, int_aarch64_sve_rev_b64>; defm REV_ZZ : sve_int_perm_reverse_z<"rev", vector_reverse>; defm SUNPKLO_ZZ : sve_int_perm_unpk<0b00, "sunpklo", AArch64sunpklo>; @@ -1672,12 +1672,12 @@ let Predicates = [HasSVEorSME] in { defm TRN1_ZZZ : sve_int_perm_bin_perm_zz<0b100, "trn1", AArch64trn1>; defm TRN2_ZZZ : sve_int_perm_bin_perm_zz<0b101, "trn2", AArch64trn2>; - defm ZIP1_PPP : sve_int_perm_bin_perm_pp<0b000, "zip1", AArch64zip1>; - defm ZIP2_PPP : sve_int_perm_bin_perm_pp<0b001, "zip2", AArch64zip2>; - defm UZP1_PPP : sve_int_perm_bin_perm_pp<0b010, "uzp1", AArch64uzp1>; - defm UZP2_PPP : sve_int_perm_bin_perm_pp<0b011, "uzp2", AArch64uzp2>; - defm TRN1_PPP : sve_int_perm_bin_perm_pp<0b100, "trn1", AArch64trn1>; - defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2", AArch64trn2>; + defm ZIP1_PPP : sve_int_perm_bin_perm_pp<0b000, "zip1", AArch64zip1, int_aarch64_sve_zip1_b16, int_aarch64_sve_zip1_b32, int_aarch64_sve_zip1_b64>; + defm ZIP2_PPP : sve_int_perm_bin_perm_pp<0b001, "zip2", AArch64zip2, int_aarch64_sve_zip2_b16, int_aarch64_sve_zip2_b32, int_aarch64_sve_zip2_b64>; + defm UZP1_PPP : sve_int_perm_bin_perm_pp<0b010, "uzp1", AArch64uzp1, int_aarch64_sve_uzp1_b16, int_aarch64_sve_uzp1_b32, int_aarch64_sve_uzp1_b64>; + defm UZP2_PPP : sve_int_perm_bin_perm_pp<0b011, "uzp2", AArch64uzp2, int_aarch64_sve_uzp2_b16, int_aarch64_sve_uzp2_b32, int_aarch64_sve_uzp2_b64>; + defm TRN1_PPP : sve_int_perm_bin_perm_pp<0b100, "trn1", AArch64trn1, int_aarch64_sve_trn1_b16, int_aarch64_sve_trn1_b32, int_aarch64_sve_trn1_b64>; + defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2", AArch64trn2, int_aarch64_sve_trn2_b16, int_aarch64_sve_trn2_b32, int_aarch64_sve_trn2_b64>; // Extract lo/hi halves of legal predicate types. def : Pat<(nxv1i1 (extract_subvector (nxv2i1 PPR:$Ps), (i64 0))), diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index cef8d41218e8..1f0626191c0d 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1448,11 +1448,12 @@ multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; } -class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> +class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty, + SDPatternOperator op> : I<(outs pprty:$Pd), (ins pprty:$Pn), asm, "\t$Pd, $Pn", "", - []>, Sched<[]> { + [(set nxv16i1:$Pd, (op nxv16i1:$Pn))]>, Sched<[]> { bits<4> Pd; bits<4> Pn; let Inst{31-24} = 0b00000101; @@ -1463,16 +1464,18 @@ class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> let Inst{3-0} = Pd; } -multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { - def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; - def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; - def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; - def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; +multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator ir_op, + SDPatternOperator op_b16, + SDPatternOperator op_b32, + SDPatternOperator op_b64> { + def _B : sve_int_perm_reverse_p<0b00, asm, PPR8, ir_op>; + def _H : sve_int_perm_reverse_p<0b01, asm, PPR16, op_b16>; + def _S : sve_int_perm_reverse_p<0b10, asm, PPR32, op_b32>; + def _D : sve_int_perm_reverse_p<0b11, asm, PPR64, op_b64>; - def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; - def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; - def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; - def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; + def : SVE_1_Op_Pat<nxv8i1, ir_op, nxv8i1, !cast<Instruction>(NAME # _H)>; + def : SVE_1_Op_Pat<nxv4i1, ir_op, nxv4i1, !cast<Instruction>(NAME # _S)>; + def : SVE_1_Op_Pat<nxv2i1, ir_op, nxv2i1, !cast<Instruction>(NAME # _D)>; } class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, @@ -6327,10 +6330,11 @@ multiclass sve_mem_p_spill<string asm> { //===----------------------------------------------------------------------===// class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, - PPRRegOp pprty> + PPRRegOp pprty, SDPatternOperator op> : I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), asm, "\t$Pd, $Pn, $Pm", - "", []>, Sched<[]> { + "", + [(set nxv16i1:$Pd, (op nxv16i1:$Pn, nxv16i1:$Pm))]>, Sched<[]> { bits<4> Pd; bits<4> Pm; bits<4> Pn; @@ -6347,16 +6351,18 @@ class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, } multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, - SDPatternOperator op> { - def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; - def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; - def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; - def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; + SDPatternOperator ir_op, + SDPatternOperator op_b16, + SDPatternOperator op_b32, + SDPatternOperator op_b64> { + def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8, ir_op>; + def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16, op_b16>; + def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32, op_b32>; + def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64, op_b64>; - def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; - def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; - def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; - def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; + def : SVE_2_Op_Pat<nxv8i1, ir_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; + def : SVE_2_Op_Pat<nxv4i1, ir_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; + def : SVE_2_Op_Pat<nxv2i1, ir_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; } class sve_int_perm_punpk<bit opc, string asm> diff --git a/llvm/lib/Target/RISCV/RISCVSExtWRemoval.cpp b/llvm/lib/Target/RISCV/RISCVSExtWRemoval.cpp index 2ee228d72825..a26a3f2411f8 100644 --- a/llvm/lib/Target/RISCV/RISCVSExtWRemoval.cpp +++ b/llvm/lib/Target/RISCV/RISCVSExtWRemoval.cpp @@ -175,8 +175,9 @@ static bool isSignExtendedW(Register SrcReg, const MachineRegisterInfo &MRI, const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs(); unsigned BitWidth = IntTy->getBitWidth(); - return (BitWidth <= 32 && Attrs.hasAttribute(Attribute::SExt)) || - (BitWidth < 32 && Attrs.hasAttribute(Attribute::ZExt)); + if ((BitWidth <= 32 && Attrs.hasAttribute(Attribute::SExt)) || + (BitWidth < 32 && Attrs.hasAttribute(Attribute::ZExt))) + continue; } if (!AddRegDefToWorkList(CopySrcReg)) diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index c92a30804014..4dd8a6cdd898 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -1154,11 +1154,11 @@ def CSR_64_CXX_TLS_Darwin_PE : CalleeSavedRegs<(add RBP)>; // CSRs that are handled explicitly via copies. def CSR_64_CXX_TLS_Darwin_ViaCopy : CalleeSavedRegs<(sub CSR_64_TLS_Darwin, RBP)>; -// All GPRs - except r11 and return registers. +// All GPRs - except r11 def CSR_64_RT_MostRegs : CalleeSavedRegs<(add CSR_64, RAX, RCX, RDX, RSI, RDI, R8, R9, R10)>; -// All registers - except r11 and return registers. +// All registers - except r11 def CSR_64_RT_AllRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs, (sequence "XMM%u", 0, 15))>; def CSR_64_RT_AllRegs_AVX : CalleeSavedRegs<(add CSR_64_RT_MostRegs, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index a33ee63c877e..cf17c51b04fc 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -104,20 +104,6 @@ static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl, DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc())); } -/// Returns true if a CC can dynamically exclude a register from the list of -/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on -/// params/returns. -static bool shouldDisableCalleeSavedRegisterCC(CallingConv::ID CC) { - switch (CC) { - default: - return false; - case CallingConv::X86_RegCall: - case CallingConv::PreserveMost: - case CallingConv::PreserveAll: - return true; - } -} - X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, const X86Subtarget &STI) : TargetLowering(TM), Subtarget(STI) { @@ -3181,7 +3167,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, // In some cases we need to disable registers from the default CSR list. // For example, when they are used for argument passing. bool ShouldDisableCalleeSavedRegister = - shouldDisableCalleeSavedRegisterCC(CallConv) || + CallConv == CallingConv::X86_RegCall || MF.getFunction().hasFnAttribute("no_caller_saved_registers"); if (CallConv == CallingConv::X86_INTR && !Outs.empty()) @@ -4333,7 +4319,7 @@ SDValue X86TargetLowering::LowerFormalArguments( } } - if (shouldDisableCalleeSavedRegisterCC(CallConv) || + if (CallConv == CallingConv::X86_RegCall || F.hasFnAttribute("no_caller_saved_registers")) { MachineRegisterInfo &MRI = MF.getRegInfo(); for (std::pair<Register, Register> Pair : MRI.liveins()) @@ -4894,7 +4880,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // In some calling conventions we need to remove the used physical registers // from the reg mask. - if (shouldDisableCalleeSavedRegisterCC(CallConv) || HasNCSR) { + if (CallConv == CallingConv::X86_RegCall || HasNCSR) { const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); // Allocate a new Reg Mask and copy Mask. @@ -22000,15 +21986,25 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, // Extend everything to 80 bits to force it to be done on x87. // TODO: Are there any fast-math-flags to propagate here? if (IsStrict) { - SDValue Add = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::f80, MVT::Other}, - {Chain, Fild, Fudge}); + unsigned Opc = ISD::STRICT_FADD; + // Windows needs the precision control changed to 80bits around this add. + if (Subtarget.isOSWindows() && DstVT == MVT::f32) + Opc = X86ISD::STRICT_FP80_ADD; + + SDValue Add = + DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge}); // STRICT_FP_ROUND can't handle equal types. if (DstVT == MVT::f80) return Add; return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other}, {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)}); } - SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); + unsigned Opc = ISD::FADD; + // Windows needs the precision control changed to 80bits around this add. + if (Subtarget.isOSWindows() && DstVT == MVT::f32) + Opc = X86ISD::FP80_ADD; + + SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge); return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); } @@ -34804,6 +34800,8 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(AESDECWIDE256KL) NODE_NAME_CASE(CMPCCXADD) NODE_NAME_CASE(TESTUI) + NODE_NAME_CASE(FP80_ADD) + NODE_NAME_CASE(STRICT_FP80_ADD) } return nullptr; #undef NODE_NAME_CASE @@ -37314,6 +37312,69 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } + case X86::FP80_ADDr: + case X86::FP80_ADDm32: { + // Change the floating point control register to use double extended + // precision when performing the addition. + int OrigCWFrameIdx = + MF->getFrameInfo().CreateStackObject(2, Align(2), false); + addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::FNSTCW16m)), + OrigCWFrameIdx); + + // Load the old value of the control word... + Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); + addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOVZX32rm16), OldCW), + OrigCWFrameIdx); + + // OR 0b11 into bit 8 and 9. 0b11 is the encoding for double extended + // precision. + Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); + BuildMI(*BB, MI, DL, TII->get(X86::OR32ri), NewCW) + .addReg(OldCW, RegState::Kill) + .addImm(0x300); + + // Extract to 16 bits. + Register NewCW16 = + MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), NewCW16) + .addReg(NewCW, RegState::Kill, X86::sub_16bit); + + // Prepare memory for FLDCW. + int NewCWFrameIdx = + MF->getFrameInfo().CreateStackObject(2, Align(2), false); + addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)), + NewCWFrameIdx) + .addReg(NewCW16, RegState::Kill); + + // Reload the modified control word now... + addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::FLDCW16m)), + NewCWFrameIdx); + + // Do the addition. + if (MI.getOpcode() == X86::FP80_ADDr) { + BuildMI(*BB, MI, DL, TII->get(X86::ADD_Fp80)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)); + } else { + BuildMI(*BB, MI, DL, TII->get(X86::ADD_Fp80m32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)) + .add(MI.getOperand(4)) + .add(MI.getOperand(5)) + .add(MI.getOperand(6)); + } + + // Reload the original control word now. + addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::FLDCW16m)), + OrigCWFrameIdx); + + MI.eraseFromParent(); // The pseudo instruction is gone now. + return BB; + } + case X86::FP32_TO_INT16_IN_MEM: case X86::FP32_TO_INT32_IN_MEM: case X86::FP32_TO_INT64_IN_MEM: diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index c5c115047271..d802d5f53aa2 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -740,6 +740,9 @@ namespace llvm { // User level interrupts - testui TESTUI, + // Perform an FP80 add after changing precision control in FPCW. + FP80_ADD, + /// X86 strict FP compare instructions. STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, STRICT_FCMPS, @@ -779,6 +782,9 @@ namespace llvm { STRICT_CVTPS2PH, STRICT_CVTPH2PS, + // Perform an FP80 add after changing precision control in FPCW. + STRICT_FP80_ADD, + // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE. diff --git a/llvm/lib/Target/X86/X86InstrFPStack.td b/llvm/lib/Target/X86/X86InstrFPStack.td index a68d61043c5c..fbbd3c83dc5c 100644 --- a/llvm/lib/Target/X86/X86InstrFPStack.td +++ b/llvm/lib/Target/X86/X86InstrFPStack.td @@ -26,6 +26,13 @@ def SDTX86Fist : SDTypeProfile<0, 2, [SDTCisFP<0>, SDTCisPtrTy<1>]>; def SDTX86CwdStore : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>; def SDTX86CwdLoad : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>; +def X86fp80_add : SDNode<"X86ISD::FP80_ADD", SDTFPBinOp, [SDNPCommutative]>; +def X86strict_fp80_add : SDNode<"X86ISD::STRICT_FP80_ADD", SDTFPBinOp, + [SDNPHasChain,SDNPCommutative]>; +def any_X86fp80_add : PatFrags<(ops node:$lhs, node:$rhs), + [(X86strict_fp80_add node:$lhs, node:$rhs), + (X86fp80_add node:$lhs, node:$rhs)]>; + def X86fld : SDNode<"X86ISD::FLD", SDTX86Fld, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def X86fst : SDNode<"X86ISD::FST", SDTX86Fst, @@ -141,6 +148,14 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [EFLAGS] in { [(X86fp_to_i32mem RFP80:$src, addr:$dst)]>; def FP80_TO_INT64_IN_MEM : PseudoI<(outs), (ins i64mem:$dst, RFP80:$src), [(X86fp_to_i64mem RFP80:$src, addr:$dst)]>; + + def FP80_ADDr : PseudoI<(outs RFP80:$dst), (ins RFP80:$src1, RFP80:$src2), + [(set RFP80:$dst, + (any_X86fp80_add RFP80:$src1, RFP80:$src2))]>; + def FP80_ADDm32 : PseudoI<(outs RFP80:$dst), (ins RFP80:$src1, f32mem:$src2), + [(set RFP80:$dst, + (any_X86fp80_add RFP80:$src1, + (f80 (extloadf32 addr:$src2))))]>; } // All FP Stack operations are represented with four instructions here. The diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 001ef55ba472..42158e4e05dd 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -1043,12 +1043,14 @@ struct AAPointerInfoImpl const auto &NoSyncAA = A.getAAFor<AANoSync>( QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL); const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>( - IRPosition::function(Scope), &QueryingAA, DepClassTy::OPTIONAL); + IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE); bool AllInSameNoSyncFn = NoSyncAA.isAssumedNoSync(); bool InstIsExecutedByInitialThreadOnly = ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I); bool InstIsExecutedInAlignedRegion = ExecDomainAA && ExecDomainAA->isExecutedInAlignedRegion(A, I); + if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly) + A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL); InformationCache &InfoCache = A.getInfoCache(); bool IsThreadLocalObj = @@ -1063,14 +1065,24 @@ struct AAPointerInfoImpl auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool { if (IsThreadLocalObj || AllInSameNoSyncFn) return true; - if (!ExecDomainAA) + const auto *FnExecDomainAA = + I.getFunction() == &Scope + ? ExecDomainAA + : A.lookupAAFor<AAExecutionDomain>( + IRPosition::function(*I.getFunction()), &QueryingAA, + DepClassTy::NONE); + if (!FnExecDomainAA) return false; if (InstIsExecutedInAlignedRegion || - ExecDomainAA->isExecutedInAlignedRegion(A, I)) + FnExecDomainAA->isExecutedInAlignedRegion(A, I)) { + A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL); return true; + } if (InstIsExecutedByInitialThreadOnly && - ExecDomainAA->isExecutedByInitialThreadOnly(I)) + FnExecDomainAA->isExecutedByInitialThreadOnly(I)) { + A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL); return true; + } return false; }; @@ -4161,12 +4173,14 @@ struct AAIsDeadFloating : public AAIsDeadValueImpl { return true; if (auto *LI = dyn_cast<LoadInst>(V)) { if (llvm::all_of(LI->uses(), [&](const Use &U) { - return InfoCache.isOnlyUsedByAssume( - cast<Instruction>(*U.getUser())) || - A.isAssumedDead(U, this, nullptr, UsedAssumedInformation); + auto &UserI = cast<Instruction>(*U.getUser()); + if (InfoCache.isOnlyUsedByAssume(UserI)) { + if (AssumeOnlyInst) + AssumeOnlyInst->insert(&UserI); + return true; + } + return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation); })) { - if (AssumeOnlyInst) - AssumeOnlyInst->insert(LI); return true; } } diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index bee154dab10f..eb499a1aa912 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -188,9 +188,9 @@ struct AAICVTracker; struct OMPInformationCache : public InformationCache { OMPInformationCache(Module &M, AnalysisGetter &AG, BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC, - KernelSet &Kernels) + KernelSet &Kernels, bool OpenMPPostLink) : InformationCache(M, AG, Allocator, CGSCC), OMPBuilder(M), - Kernels(Kernels) { + Kernels(Kernels), OpenMPPostLink(OpenMPPostLink) { OMPBuilder.initialize(); initializeRuntimeFunctions(M); @@ -448,6 +448,24 @@ struct OMPInformationCache : public InformationCache { CI->setCallingConv(Fn->getCallingConv()); } + // Helper function to determine if it's legal to create a call to the runtime + // functions. + bool runtimeFnsAvailable(ArrayRef<RuntimeFunction> Fns) { + // We can always emit calls if we haven't yet linked in the runtime. + if (!OpenMPPostLink) + return true; + + // Once the runtime has been already been linked in we cannot emit calls to + // any undefined functions. + for (RuntimeFunction Fn : Fns) { + RuntimeFunctionInfo &RFI = RFIs[Fn]; + + if (RFI.Declaration && RFI.Declaration->isDeclaration()) + return false; + } + return true; + } + /// Helper to initialize all runtime function information for those defined /// in OpenMPKinds.def. void initializeRuntimeFunctions(Module &M) { @@ -523,6 +541,9 @@ struct OMPInformationCache : public InformationCache { /// Collection of known OpenMP runtime functions.. DenseSet<const Function *> RTLFunctions; + + /// Indicates if we have already linked in the OpenMP device library. + bool OpenMPPostLink = false; }; template <typename Ty, bool InsertInvalidates = true> @@ -1412,7 +1433,10 @@ private: Changed |= WasSplit; return WasSplit; }; - RFI.foreachUse(SCC, SplitMemTransfers); + if (OMPInfoCache.runtimeFnsAvailable( + {OMPRTL___tgt_target_data_begin_mapper_issue, + OMPRTL___tgt_target_data_begin_mapper_wait})) + RFI.foreachUse(SCC, SplitMemTransfers); return Changed; } @@ -2656,7 +2680,9 @@ struct AAExecutionDomainFunction : public AAExecutionDomain { bool isExecutedInAlignedRegion(Attributor &A, const Instruction &I) const override { - if (!isValidState() || isa<CallBase>(I)) + assert(I.getFunction() == getAnchorScope() && + "Instruction is out of scope!"); + if (!isValidState()) return false; const Instruction *CurI; @@ -2667,14 +2693,18 @@ struct AAExecutionDomainFunction : public AAExecutionDomain { auto *CB = dyn_cast<CallBase>(CurI); if (!CB) continue; + if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) { + break; + } const auto &It = CEDMap.find(CB); if (It == CEDMap.end()) continue; - if (!It->getSecond().IsReachedFromAlignedBarrierOnly) + if (!It->getSecond().IsReachingAlignedBarrierOnly) return false; + break; } while ((CurI = CurI->getNextNonDebugInstruction())); - if (!CurI && !BEDMap.lookup(I.getParent()).IsReachedFromAlignedBarrierOnly) + if (!CurI && !BEDMap.lookup(I.getParent()).IsReachingAlignedBarrierOnly) return false; // Check backward until a call or the block beginning is reached. @@ -2683,12 +2713,16 @@ struct AAExecutionDomainFunction : public AAExecutionDomain { auto *CB = dyn_cast<CallBase>(CurI); if (!CB) continue; + if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) { + break; + } const auto &It = CEDMap.find(CB); if (It == CEDMap.end()) continue; if (!AA::isNoSyncInst(A, *CB, *this)) { - if (It->getSecond().IsReachedFromAlignedBarrierOnly) + if (It->getSecond().IsReachedFromAlignedBarrierOnly) { break; + } return false; } @@ -2984,7 +3018,8 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) { if (EDAA.getState().isValidState()) { const auto &CalleeED = EDAA.getFunctionExecutionDomain(); ED.IsReachedFromAlignedBarrierOnly = - CalleeED.IsReachedFromAlignedBarrierOnly; + CallED.IsReachedFromAlignedBarrierOnly = + CalleeED.IsReachedFromAlignedBarrierOnly; AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly; if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly) ED.EncounteredNonLocalSideEffect |= @@ -2999,8 +3034,9 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) { continue; } } - ED.IsReachedFromAlignedBarrierOnly = - IsNoSync && ED.IsReachedFromAlignedBarrierOnly; + if (!IsNoSync) + ED.IsReachedFromAlignedBarrierOnly = + CallED.IsReachedFromAlignedBarrierOnly = false; AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly; ED.EncounteredNonLocalSideEffect |= !CB->doesNotAccessMemory(); if (!IsNoSync) @@ -3914,6 +3950,12 @@ struct AAKernelInfoFunction : AAKernelInfo { bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) { auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache()); + // We cannot change to SPMD mode if the runtime functions aren't availible. + if (!OMPInfoCache.runtimeFnsAvailable( + {OMPRTL___kmpc_get_hardware_thread_id_in_block, + OMPRTL___kmpc_barrier_simple_spmd})) + return false; + if (!SPMDCompatibilityTracker.isAssumed()) { for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) { if (!NonCompatibleI) @@ -4021,6 +4063,13 @@ struct AAKernelInfoFunction : AAKernelInfo { if (!ReachedKnownParallelRegions.isValidState()) return ChangeStatus::UNCHANGED; + auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache()); + if (!OMPInfoCache.runtimeFnsAvailable( + {OMPRTL___kmpc_get_hardware_num_threads_in_block, + OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic, + OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel})) + return ChangeStatus::UNCHANGED; + const int InitModeArgNo = 1; const int InitUseStateMachineArgNo = 2; @@ -4167,7 +4216,6 @@ struct AAKernelInfoFunction : AAKernelInfo { BranchInst::Create(IsWorkerCheckBB, UserCodeEntryBB, IsWorker, InitBB); Module &M = *Kernel->getParent(); - auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache()); FunctionCallee BlockHwSizeFn = OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction( M, OMPRTL___kmpc_get_hardware_num_threads_in_block); @@ -5343,7 +5391,10 @@ PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) { BumpPtrAllocator Allocator; CallGraphUpdater CGUpdater; - OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, Kernels); + bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink || + LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink; + OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, Kernels, + PostLink); unsigned MaxFixpointIterations = (isOpenMPDevice(M)) ? SetFixpointIterations : 32; @@ -5417,9 +5468,11 @@ PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C, CallGraphUpdater CGUpdater; CGUpdater.initialize(CG, C, AM, UR); + bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink || + LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink; SetVector<Function *> Functions(SCC.begin(), SCC.end()); OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator, - /*CGSCC*/ &Functions, Kernels); + /*CGSCC*/ &Functions, Kernels, PostLink); unsigned MaxFixpointIterations = (isOpenMPDevice(M)) ? SetFixpointIterations : 32; diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp index 2b2eda5d8587..7366059cd242 100644 --- a/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -23,6 +23,10 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" +#include "llvm/Debuginfod/BuildIDFetcher.h" +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Debuginfod/HTTPClient.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/CommandLine.h" @@ -179,6 +183,8 @@ private: /// Allowlist from -name-allowlist to be used for filtering. std::unique_ptr<SpecialCaseList> NameAllowlist; + + std::unique_ptr<object::BuildIDFetcher> BIDFetcher; }; } @@ -435,7 +441,7 @@ std::unique_ptr<CoverageMapping> CodeCoverageTool::load() { ObjectFilename); auto CoverageOrErr = CoverageMapping::load(ObjectFilenames, PGOFilename, CoverageArches, - ViewOpts.CompilationDirectory); + ViewOpts.CompilationDirectory, BIDFetcher.get()); if (Error E = CoverageOrErr.takeError()) { error("Failed to load coverage: " + toString(std::move(E))); return nullptr; @@ -647,6 +653,14 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { cl::opt<bool> DebugDump("dump", cl::Optional, cl::desc("Show internal debug dump")); + cl::list<std::string> DebugFileDirectory( + "debug-file-directory", + cl::desc("Directories to search for object files by build ID")); + cl::opt<bool> Debuginfod( + "debuginfod", cl::ZeroOrMore, + cl::desc("Use debuginfod to look up object files from profile"), + cl::init(canUseDebuginfod())); + cl::opt<CoverageViewOptions::OutputFormat> Format( "format", cl::desc("Output format for line-based coverage reports"), cl::values(clEnumValN(CoverageViewOptions::OutputFormat::Text, "text", @@ -749,12 +763,18 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { auto commandLineParser = [&, this](int argc, const char **argv) -> int { cl::ParseCommandLineOptions(argc, argv, "LLVM code coverage tool\n"); ViewOpts.Debug = DebugDump; + if (Debuginfod) { + HTTPClient::initialize(); + BIDFetcher = std::make_unique<DebuginfodFetcher>(DebugFileDirectory); + } else { + BIDFetcher = std::make_unique<object::BuildIDFetcher>(DebugFileDirectory); + } if (!CovFilename.empty()) ObjectFilenames.emplace_back(CovFilename); for (const std::string &Filename : CovFilenames) ObjectFilenames.emplace_back(Filename); - if (ObjectFilenames.empty()) { + if (ObjectFilenames.empty() && !Debuginfod && DebugFileDirectory.empty()) { errs() << "No filenames specified!\n"; ::exit(1); } @@ -867,10 +887,8 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { } CoverageArches.emplace_back(Arch); } - if (CoverageArches.size() == 1) - CoverageArches.insert(CoverageArches.end(), ObjectFilenames.size() - 1, - CoverageArches[0]); - if (CoverageArches.size() != ObjectFilenames.size()) { + if (CoverageArches.size() != 1 && + CoverageArches.size() != ObjectFilenames.size()) { error("Number of architectures doesn't match the number of objects"); return 1; } diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index 930b132533cd..9979a26cf115 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -3198,9 +3198,7 @@ int main(int argc, char **argv) { // Initialize debuginfod. const bool ShouldUseDebuginfodByDefault = - InputArgs.hasArg(OBJDUMP_build_id) || - (HTTPClient::isAvailable() && - !ExitOnErr(getDefaultDebuginfodUrls()).empty()); + InputArgs.hasArg(OBJDUMP_build_id) || canUseDebuginfod(); std::vector<std::string> DebugFileDirectories = InputArgs.getAllArgValues(OBJDUMP_debug_file_directory); if (InputArgs.hasFlag(OBJDUMP_debuginfod, OBJDUMP_no_debuginfod, diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp index 1b86134dda51..ed24e8550291 100644 --- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp +++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp @@ -443,13 +443,7 @@ int main(int argc, char **argv) { LLVMSymbolizer Symbolizer(Opts); - // A debuginfod lookup could succeed if a HTTP client is available and at - // least one backing URL is configured. - bool ShouldUseDebuginfodByDefault = - HTTPClient::isAvailable() && - !ExitOnErr(getDefaultDebuginfodUrls()).empty(); - if (Args.hasFlag(OPT_debuginfod, OPT_no_debuginfod, - ShouldUseDebuginfodByDefault)) + if (Args.hasFlag(OPT_debuginfod, OPT_no_debuginfod, canUseDebuginfod())) enableDebuginfod(Symbolizer, Args); if (Args.hasArg(OPT_filter_markup)) { |