diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp | 750 |
1 files changed, 446 insertions, 304 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp index eb13cd40eb8a..63ed5b4dd0c3 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp @@ -30,6 +30,7 @@ #include "clang/AST/ASTContext.h" #include "clang/AST/ASTLambda.h" #include "clang/AST/CharUnits.h" +#include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/DeclTemplate.h" @@ -52,6 +53,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/AttributeMask.h" #include "llvm/IR/CallingConv.h" @@ -67,11 +69,12 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/RISCVISAInfo.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/xxhash.h" +#include "llvm/TargetParser/RISCVISAInfo.h" #include "llvm/TargetParser/Triple.h" #include "llvm/TargetParser/X86TargetParser.h" +#include "llvm/Transforms/Utils/BuildLibCalls.h" #include <optional> using namespace clang; @@ -144,6 +147,10 @@ createTargetCodeGenInfo(CodeGenModule &CGM) { Kind = AArch64ABIKind::DarwinPCS; else if (Triple.isOSWindows()) return createWindowsAArch64TargetCodeGenInfo(CGM, AArch64ABIKind::Win64); + else if (Target.getABI() == "aapcs-soft") + Kind = AArch64ABIKind::AAPCSSoft; + else if (Target.getABI() == "pauthtest") + Kind = AArch64ABIKind::PAuthTest; return createAArch64TargetCodeGenInfo(CGM, Kind); } @@ -363,7 +370,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth()); IntPtrTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getMaxPointerWidth()); - Int8PtrTy = llvm::PointerType::get(LLVMContext, 0); + Int8PtrTy = llvm::PointerType::get(LLVMContext, + C.getTargetAddressSpace(LangAS::Default)); const llvm::DataLayout &DL = M.getDataLayout(); AllocaInt8PtrTy = llvm::PointerType::get(LLVMContext, DL.getAllocaAddrSpace()); @@ -396,8 +404,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0. if (LangOpts.Sanitize.has(SanitizerKind::Thread) || (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0)) - TBAA.reset(new CodeGenTBAA(Context, TheModule, CodeGenOpts, getLangOpts(), - getCXXABI().getMangleContext())); + TBAA.reset(new CodeGenTBAA(Context, getTypes(), TheModule, CodeGenOpts, + getLangOpts(), getCXXABI().getMangleContext())); // If debug info or coverage generation is enabled, create the CGDebugInfo // object. @@ -439,6 +447,11 @@ CodeGenModule::CodeGenModule(ASTContext &C, } ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(Path); } + + // Record mregparm value now so it is visible through all of codegen. + if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86) + getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters", + CodeGenOpts.NumRegisterParameters); } CodeGenModule::~CodeGenModule() {} @@ -625,6 +638,26 @@ static bool checkAliasedGlobal( return true; } +// Emit a warning if toc-data attribute is requested for global variables that +// have aliases and remove the toc-data attribute. +static void checkAliasForTocData(llvm::GlobalVariable *GVar, + const CodeGenOptions &CodeGenOpts, + DiagnosticsEngine &Diags, + SourceLocation Location) { + if (GVar->hasAttribute("toc-data")) { + auto GVId = GVar->getName(); + // Is this a global variable specified by the user as local? + if ((llvm::binary_search(CodeGenOpts.TocDataVarsUserSpecified, GVId))) { + Diags.Report(Location, diag::warn_toc_unsupported_type) + << GVId << "the variable has an alias"; + } + llvm::AttributeSet CurrAttributes = GVar->getAttributes(); + llvm::AttributeSet NewAttributes = + CurrAttributes.removeAttribute(GVar->getContext(), "toc-data"); + GVar->setAttributes(NewAttributes); + } +} + void CodeGenModule::checkAliases() { // Check if the constructed aliases are well formed. It is really unfortunate // that we have to do this in CodeGen, but we only construct mangled names @@ -651,6 +684,12 @@ void CodeGenModule::checkAliases() { continue; } + if (getContext().getTargetInfo().getTriple().isOSAIX()) + if (const llvm::GlobalVariable *GVar = + dyn_cast<const llvm::GlobalVariable>(GV)) + checkAliasForTocData(const_cast<llvm::GlobalVariable *>(GVar), + getCodeGenOpts(), Diags, Location); + llvm::Constant *Aliasee = IsIFunc ? cast<llvm::GlobalIFunc>(Alias)->getResolver() : cast<llvm::GlobalAlias>(Alias)->getAliasee(); @@ -686,6 +725,11 @@ void CodeGenModule::checkAliases() { cast<llvm::GlobalAlias>(Alias)->setAliasee(Aliasee); } } + // ifunc resolvers are usually implemented to run before sanitizer + // initialization. Disable instrumentation to prevent the ordering issue. + if (IsIFunc) + cast<llvm::Function>(Aliasee)->addFnAttr( + llvm::Attribute::DisableSanitizerInstrumentation); } if (!Error) return; @@ -837,10 +881,6 @@ void CodeGenModule::Release() { AddGlobalCtor(CudaCtorFunction); } if (OpenMPRuntime) { - if (llvm::Function *OpenMPRequiresDirectiveRegFun = - OpenMPRuntime->emitRequiresDirectiveRegFun()) { - AddGlobalCtor(OpenMPRequiresDirectiveRegFun, 0); - } OpenMPRuntime->createOffloadEntriesAndInfoMetadata(); OpenMPRuntime->clear(); } @@ -861,6 +901,7 @@ void CodeGenModule::Release() { checkAliases(); EmitDeferredUnusedCoverageMappings(); CodeGenPGO(*this).setValueProfilingFlag(getModule()); + CodeGenPGO(*this).setProfileVersion(getModule()); if (CoverageMapping) CoverageMapping->emit(); if (CodeGenOpts.SanitizeCfiCrossDso) { @@ -873,13 +914,14 @@ void CodeGenModule::Release() { if (Context.getTargetInfo().getTriple().isWasm()) EmitMainVoidAlias(); - if (getTriple().isAMDGPU()) { - // Emit amdgpu_code_object_version module flag, which is code object version + if (getTriple().isAMDGPU() || + (getTriple().isSPIRV() && getTriple().getVendor() == llvm::Triple::AMD)) { + // Emit amdhsa_code_object_version module flag, which is code object version // times 100. if (getTarget().getTargetOpts().CodeObjectVersion != llvm::CodeObjectVersionKind::COV_None) { getModule().addModuleFlag(llvm::Module::Error, - "amdgpu_code_object_version", + "amdhsa_code_object_version", getTarget().getTargetOpts().CodeObjectVersion); } @@ -918,7 +960,15 @@ void CodeGenModule::Release() { llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external"); addCompilerUsedGlobal(GV); } - + if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) { + // Emit a unique ID so that host and device binaries from the same + // compilation unit can be associated. + auto *GV = new llvm::GlobalVariable( + getModule(), Int8Ty, false, llvm::GlobalValue::ExternalLinkage, + llvm::Constant::getNullValue(Int8Ty), + "__hip_cuid_" + getContext().getCUIDHash()); + addCompilerUsedGlobal(GV); + } emitLLVMUsed(); if (SanStats) SanStats->finish(); @@ -946,11 +996,6 @@ void CodeGenModule::Release() { NMD->addOperand(MD); } - // Record mregparm value now so it is visible through rest of codegen. - if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86) - getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters", - CodeGenOpts.NumRegisterParameters); - if (CodeGenOpts.DwarfVersion) { getModule().addModuleFlag(llvm::Module::Max, "Dwarf Version", CodeGenOpts.DwarfVersion); @@ -1045,17 +1090,14 @@ void CodeGenModule::Release() { llvm::MDString::get(VMContext, "ascii")); } - llvm::Triple::ArchType Arch = Context.getTargetInfo().getTriple().getArch(); - if ( Arch == llvm::Triple::arm - || Arch == llvm::Triple::armeb - || Arch == llvm::Triple::thumb - || Arch == llvm::Triple::thumbeb) { + llvm::Triple T = Context.getTargetInfo().getTriple(); + if (T.isARM() || T.isThumb()) { // The minimum width of an enum in bytes uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4; getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth); } - if (Arch == llvm::Triple::riscv32 || Arch == llvm::Triple::riscv64) { + if (T.isRISCV()) { StringRef ABIStr = Target.getABI(); llvm::LLVMContext &Ctx = TheModule.getContext(); getModule().addModuleFlag(llvm::Module::Error, "target-abi", @@ -1066,8 +1108,8 @@ void CodeGenModule::Release() { // unique ISA strings that were linked together. const std::vector<std::string> &Features = getTarget().getTargetOpts().Features; - auto ParseResult = llvm::RISCVISAInfo::parseFeatures( - Arch == llvm::Triple::riscv64 ? 64 : 32, Features); + auto ParseResult = + llvm::RISCVISAInfo::parseFeatures(T.isRISCV64() ? 64 : 32, Features); if (!errorToBool(ParseResult.takeError())) getModule().addModuleFlag( llvm::Module::AppendUnique, "riscv-isa", @@ -1141,10 +1183,7 @@ void CodeGenModule::Release() { getModule().addModuleFlag(llvm::Module::Override, "tag-stack-memory-buildattr", 1); - if (Arch == llvm::Triple::thumb || Arch == llvm::Triple::thumbeb || - Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb || - Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 || - Arch == llvm::Triple::aarch64_be) { + if (T.isARM() || T.isThumb() || T.isAArch64()) { if (LangOpts.BranchTargetEnforcement) getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement", 1); @@ -1161,6 +1200,37 @@ void CodeGenModule::Release() { if (!LangOpts.isSignReturnAddressWithAKey()) getModule().addModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 1); + + if (getTriple().isOSLinux()) { + assert(getTriple().isOSBinFormatELF()); + using namespace llvm::ELF; + uint64_t PAuthABIVersion = + (LangOpts.PointerAuthIntrinsics + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INTRINSICS) | + (LangOpts.PointerAuthCalls + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_CALLS) | + (LangOpts.PointerAuthReturns + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_RETURNS) | + (LangOpts.PointerAuthAuthTraps + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_AUTHTRAPS) | + (LangOpts.PointerAuthVTPtrAddressDiscrimination + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRADDRDISCR) | + (LangOpts.PointerAuthVTPtrTypeDiscrimination + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRTYPEDISCR) | + (LangOpts.PointerAuthInitFini + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI); + static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI == + AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST, + "Update when new enum items are defined"); + if (PAuthABIVersion != 0) { + getModule().addModuleFlag(llvm::Module::Error, + "aarch64-elf-pauthabi-platform", + AARCH64_PAUTH_PLATFORM_LLVM_LINUX); + getModule().addModuleFlag(llvm::Module::Error, + "aarch64-elf-pauthabi-version", + PAuthABIVersion); + } + } } if (CodeGenOpts.StackClashProtector) @@ -1266,6 +1336,9 @@ void CodeGenModule::Release() { case CodeGenOptions::FramePointerKind::None: // 0 ("none") is the default. break; + case CodeGenOptions::FramePointerKind::Reserved: + getModule().setFramePointer(llvm::FramePointerKind::Reserved); + break; case CodeGenOptions::FramePointerKind::NonLeaf: getModule().setFramePointer(llvm::FramePointerKind::NonLeaf); break; @@ -1328,22 +1401,45 @@ void CodeGenModule::Release() { // that might affect the DLL storage class or the visibility, and // before anything that might act on these. setVisibilityFromDLLStorageClass(LangOpts, getModule()); + + // Check the tail call symbols are truly undefined. + if (getTriple().isPPC() && !MustTailCallUndefinedGlobals.empty()) { + for (auto &I : MustTailCallUndefinedGlobals) { + if (!I.first->isDefined()) + getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2; + else { + StringRef MangledName = getMangledName(GlobalDecl(I.first)); + llvm::GlobalValue *Entry = GetGlobalValue(MangledName); + if (!Entry || Entry->isWeakForLinker() || + Entry->isDeclarationForLinker()) + getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2; + } + } + } } void CodeGenModule::EmitOpenCLMetadata() { // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the // opencl.ocl.version named metadata node. - // C++ for OpenCL has a distinct mapping for versions compatibile with OpenCL. - auto Version = LangOpts.getOpenCLCompatibleVersion(); - llvm::Metadata *OCLVerElts[] = { - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( - Int32Ty, Version / 100)), - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( - Int32Ty, (Version % 100) / 10))}; - llvm::NamedMDNode *OCLVerMD = - TheModule.getOrInsertNamedMetadata("opencl.ocl.version"); - llvm::LLVMContext &Ctx = TheModule.getContext(); - OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts)); + // C++ for OpenCL has a distinct mapping for versions compatible with OpenCL. + auto CLVersion = LangOpts.getOpenCLCompatibleVersion(); + + auto EmitVersion = [this](StringRef MDName, int Version) { + llvm::Metadata *OCLVerElts[] = { + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int32Ty, Version / 100)), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int32Ty, (Version % 100) / 10))}; + llvm::NamedMDNode *OCLVerMD = TheModule.getOrInsertNamedMetadata(MDName); + llvm::LLVMContext &Ctx = TheModule.getContext(); + OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts)); + }; + + EmitVersion("opencl.ocl.version", CLVersion); + if (LangOpts.OpenCLCPlusPlus) { + // In addition to the OpenCL compatible version, emit the C++ version. + EmitVersion("opencl.cxx.version", LangOpts.OpenCLCPlusPlusVersion); + } } void CodeGenModule::EmitBackendOptionsMetadata( @@ -1727,59 +1823,6 @@ static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM, Out << ".resolver"; } -static void AppendTargetVersionMangling(const CodeGenModule &CGM, - const TargetVersionAttr *Attr, - raw_ostream &Out) { - if (Attr->isDefaultVersion()) { - Out << ".default"; - return; - } - Out << "._"; - const TargetInfo &TI = CGM.getTarget(); - llvm::SmallVector<StringRef, 8> Feats; - Attr->getFeatures(Feats); - llvm::stable_sort(Feats, [&TI](const StringRef FeatL, const StringRef FeatR) { - return TI.multiVersionSortPriority(FeatL) < - TI.multiVersionSortPriority(FeatR); - }); - for (const auto &Feat : Feats) { - Out << 'M'; - Out << Feat; - } -} - -static void AppendTargetMangling(const CodeGenModule &CGM, - const TargetAttr *Attr, raw_ostream &Out) { - if (Attr->isDefaultVersion()) - return; - - Out << '.'; - const TargetInfo &Target = CGM.getTarget(); - ParsedTargetAttr Info = Target.parseTargetAttr(Attr->getFeaturesStr()); - llvm::sort(Info.Features, [&Target](StringRef LHS, StringRef RHS) { - // Multiversioning doesn't allow "no-${feature}", so we can - // only have "+" prefixes here. - assert(LHS.starts_with("+") && RHS.starts_with("+") && - "Features should always have a prefix."); - return Target.multiVersionSortPriority(LHS.substr(1)) > - Target.multiVersionSortPriority(RHS.substr(1)); - }); - - bool IsFirst = true; - - if (!Info.CPU.empty()) { - IsFirst = false; - Out << "arch_" << Info.CPU; - } - - for (StringRef Feat : Info.Features) { - if (!IsFirst) - Out << '_'; - IsFirst = false; - Out << Feat.substr(1); - } -} - // Returns true if GD is a function decl with internal linkage and // needs a unique suffix after the mangled name. static bool isUniqueInternalLinkageDecl(GlobalDecl GD, @@ -1789,41 +1832,6 @@ static bool isUniqueInternalLinkageDecl(GlobalDecl GD, (CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage); } -static void AppendTargetClonesMangling(const CodeGenModule &CGM, - const TargetClonesAttr *Attr, - unsigned VersionIndex, - raw_ostream &Out) { - const TargetInfo &TI = CGM.getTarget(); - if (TI.getTriple().isAArch64()) { - StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); - if (FeatureStr == "default") { - Out << ".default"; - return; - } - Out << "._"; - SmallVector<StringRef, 8> Features; - FeatureStr.split(Features, "+"); - llvm::stable_sort(Features, - [&TI](const StringRef FeatL, const StringRef FeatR) { - return TI.multiVersionSortPriority(FeatL) < - TI.multiVersionSortPriority(FeatR); - }); - for (auto &Feat : Features) { - Out << 'M'; - Out << Feat; - } - } else { - Out << '.'; - StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); - if (FeatureStr.starts_with("arch=")) - Out << "arch_" << FeatureStr.substr(sizeof("arch=") - 1); - else - Out << FeatureStr; - - Out << '.' << Attr->getMangledIndex(VersionIndex); - } -} - static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, const NamedDecl *ND, bool OmitMultiVersionMangling = false) { @@ -1877,16 +1885,31 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, FD->getAttr<CPUSpecificAttr>(), GD.getMultiVersionIndex(), Out); break; - case MultiVersionKind::Target: - AppendTargetMangling(CGM, FD->getAttr<TargetAttr>(), Out); + case MultiVersionKind::Target: { + auto *Attr = FD->getAttr<TargetAttr>(); + assert(Attr && "Expected TargetAttr to be present " + "for attribute mangling"); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Out); break; - case MultiVersionKind::TargetVersion: - AppendTargetVersionMangling(CGM, FD->getAttr<TargetVersionAttr>(), Out); + } + case MultiVersionKind::TargetVersion: { + auto *Attr = FD->getAttr<TargetVersionAttr>(); + assert(Attr && "Expected TargetVersionAttr to be present " + "for attribute mangling"); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Out); break; - case MultiVersionKind::TargetClones: - AppendTargetClonesMangling(CGM, FD->getAttr<TargetClonesAttr>(), - GD.getMultiVersionIndex(), Out); + } + case MultiVersionKind::TargetClones: { + auto *Attr = FD->getAttr<TargetClonesAttr>(); + assert(Attr && "Expected TargetClonesAttr to be present " + "for attribute mangling"); + unsigned Index = GD.getMultiVersionIndex(); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Index, Out); break; + } case MultiVersionKind::None: llvm_unreachable("None multiversion type isn't valid here"); } @@ -2138,6 +2161,14 @@ void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD, llvm::AttributeList PAL; ConstructAttributeList(F->getName(), Info, GD, PAL, CallingConv, /*AttrOnCallSite=*/false, IsThunk); + if (CallingConv == llvm::CallingConv::X86_VectorCall && + getTarget().getTriple().isWindowsArm64EC()) { + SourceLocation Loc; + if (const Decl *D = GD.getDecl()) + Loc = D->getLocation(); + + Error(Loc, "__vectorcall calling convention is not currently supported"); + } F->setAttributes(PAL); F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); } @@ -2428,7 +2459,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (auto *Attr = D->getAttr<ArmNewAttr>()) { if (Attr->isNewZA()) - B.addAttribute("aarch64_pstate_za_new"); + B.addAttribute("aarch64_new_za"); if (Attr->isNewZT0()) B.addAttribute("aarch64_new_zt0"); } @@ -2678,7 +2709,7 @@ void CodeGenModule::setNonAliasAttributes(GlobalDecl GD, addUsedGlobal(F); if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>()) if (!D->getAttr<SectionAttr>()) - F->addFnAttr("implicit-section-name", SA->getName()); + F->setSection(SA->getName()); llvm::AttrBuilder Attrs(F->getContext()); if (GetCPUAndFeaturesAttributes(GD, Attrs)) { @@ -3500,6 +3531,9 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // Implicit template instantiations may change linkage if they are later // explicitly instantiated, so they should not be emitted eagerly. return false; + // Defer until all versions have been semantically checked. + if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion()) + return false; } if (const auto *VD = dyn_cast<VarDecl>(Global)) { if (Context.getInlineVariableDefinitionKind(VD) == @@ -3675,6 +3709,19 @@ template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) { return D->isImplicit(); } +bool CodeGenModule::shouldEmitCUDAGlobalVar(const VarDecl *Global) const { + assert(LangOpts.CUDA && "Should not be called by non-CUDA languages"); + // We need to emit host-side 'shadows' for all global + // device-side variables because the CUDA runtime needs their + // size and host-side address in order to provide access to + // their device-side incarnations. + return !LangOpts.CUDAIsDevice || Global->hasAttr<CUDADeviceAttr>() || + Global->hasAttr<CUDAConstantAttr>() || + Global->hasAttr<CUDASharedAttr>() || + Global->getType()->isCUDADeviceBuiltinSurfaceType() || + Global->getType()->isCUDADeviceBuiltinTextureType(); +} + void CodeGenModule::EmitGlobal(GlobalDecl GD) { const auto *Global = cast<ValueDecl>(GD.getDecl()); @@ -3699,36 +3746,27 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Non-constexpr non-lambda implicit host device functions are not emitted // unless they are used on device side. if (LangOpts.CUDA) { - if (LangOpts.CUDAIsDevice) { + assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) && + "Expected Variable or Function"); + if (const auto *VD = dyn_cast<VarDecl>(Global)) { + if (!shouldEmitCUDAGlobalVar(VD)) + return; + } else if (LangOpts.CUDAIsDevice) { const auto *FD = dyn_cast<FunctionDecl>(Global); if ((!Global->hasAttr<CUDADeviceAttr>() || - (LangOpts.OffloadImplicitHostDeviceTemplates && FD && + (LangOpts.OffloadImplicitHostDeviceTemplates && hasImplicitAttr<CUDAHostAttr>(FD) && hasImplicitAttr<CUDADeviceAttr>(FD) && !FD->isConstexpr() && !isLambdaCallOperator(FD) && !getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(FD))) && !Global->hasAttr<CUDAGlobalAttr>() && - !Global->hasAttr<CUDAConstantAttr>() && - !Global->hasAttr<CUDASharedAttr>() && - !Global->getType()->isCUDADeviceBuiltinSurfaceType() && - !Global->getType()->isCUDADeviceBuiltinTextureType() && !(LangOpts.HIPStdPar && isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>())) return; - } else { - // We need to emit host-side 'shadows' for all global - // device-side variables because the CUDA runtime needs their - // size and host-side address in order to provide access to - // their device-side incarnations. - - // So device-only functions are the only things we skip. - if (isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>() && - Global->hasAttr<CUDADeviceAttr>()) - return; - - assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) && - "Expected Variable or Function"); - } + // Device-only functions are the only things we skip. + } else if (!Global->hasAttr<CUDAHostAttr>() && + Global->hasAttr<CUDADeviceAttr>()) + return; } if (LangOpts.OpenMP) { @@ -3759,7 +3797,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + (!FD->isMultiVersion() || !getTarget().getTriple().isAArch64())) return; StringRef MangledName = getMangledName(GD); @@ -3990,9 +4029,20 @@ bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) { // behavior may break ABI compatibility of the current unit. if (const Module *M = F->getOwningModule(); M && M->getTopLevelModule()->isNamedModule() && - getContext().getCurrentNamedModule() != M->getTopLevelModule() && - !F->hasAttr<AlwaysInlineAttr>()) - return false; + getContext().getCurrentNamedModule() != M->getTopLevelModule()) { + // There are practices to mark template member function as always-inline + // and mark the template as extern explicit instantiation but not give + // the definition for member function. So we have to emit the function + // from explicitly instantiation with always-inline. + // + // See https://github.com/llvm/llvm-project/issues/86893 for details. + // + // TODO: Maybe it is better to give it a warning if we call a non-inline + // function from other module units which is marked as always-inline. + if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) { + return false; + } + } if (F->hasAttr<NoInlineAttr>()) return false; @@ -4042,17 +4092,21 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD, auto *Spec = FD->getAttr<CPUSpecificAttr>(); for (unsigned I = 0; I < Spec->cpus_size(); ++I) EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); - } else if (FD->isTargetClonesMultiVersion()) { - auto *Clone = FD->getAttr<TargetClonesAttr>(); - for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I) - if (Clone->isFirstOfVersion(I)) + } else if (auto *TC = FD->getAttr<TargetClonesAttr>()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) + // AArch64 favors the default target version over the clone if any. + if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) && + TC->isFirstOfVersion(I)) EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); // Ensure that the resolver function is also emitted. GetOrCreateMultiVersionResolver(GD); - } else if (FD->hasAttr<TargetVersionAttr>()) { - GetOrCreateMultiVersionResolver(GD); } else EmitGlobalFunctionDefinition(GD, GV); + + // Defer the resolver emission until we can reason whether the TU + // contains a default target version implementation. + if (FD->isTargetVersionMultiVersion()) + AddDeferredMultiVersionResolverToEmit(GD); } void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) { @@ -4145,105 +4199,92 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options; - if (FD->isTargetMultiVersion()) { - getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr<TargetAttr>(); - llvm::SmallVector<StringRef, 8> Feats; - TA->getAddedFeatures(Feats); - Options.emplace_back(cast<llvm::Function>(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr<TargetVersionAttr>(); - llvm::SmallVector<StringRef, 8> Feats; - TVA->getFeatures(Feats); - Options.emplace_back(cast<llvm::Function>(Func), - /*Architecture*/ "", Feats); - } - }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr<TargetClonesAttr>(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) - continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); } + assert(Func && "This should have just been created"); + } + return cast<llvm::Function>(Func); + }; - StringRef Architecture; - llvm::SmallVector<StringRef, 1> Feature; + // For AArch64, a resolver is only emitted if a function marked with + // target_version("default")) or target_clones() is present and defined + // in this TU. For other architectures it is always emitted. + bool ShouldEmitResolver = !getTarget().getTriple().isAArch64(); + SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options; - if (getTarget().getTriple().isAArch64()) { - if (Version != "default") { - llvm::SmallVector<StringRef, 8> VerFeats; - Version.split(VerFeats, "+"); - for (auto &CurFeat : VerFeats) - Feature.push_back(CurFeat.trim()); - } - } else { - if (Version.starts_with("arch=")) - Architecture = Version.drop_front(sizeof("arch=") - 1); - else if (Version != "default") - Feature.push_back(Version); - } + getContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector<StringRef, 8> Feats; + bool IsDefined = CurFD->doesThisDeclarationHaveABody(); + + if (const auto *TA = CurFD->getAttr<TargetAttr>()) { + TA->getAddedFeatures(Feats); + llvm::Function *Func = createFunction(CurFD); + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) { + if (TVA->isDefaultVersion() && IsDefined) + ShouldEmitResolver = true; + TVA->getFeatures(Feats); + llvm::Function *Func = createFunction(CurFD); + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) { + if (IsDefined) + ShouldEmitResolver = true; + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) + continue; + + llvm::Function *Func = createFunction(CurFD, I); + StringRef Architecture; + Feats.clear(); + if (getTarget().getTriple().isAArch64()) + TC->getFeatures(Feats, I); + else { + StringRef Version = TC->getFeatureStr(I); + if (Version.starts_with("arch=")) + Architecture = Version.drop_front(sizeof("arch=") - 1); + else if (Version != "default") + Feats.push_back(Version); + } + Options.emplace_back(Func, Architecture, Feats); + } + } else + llvm_unreachable("unexpected MultiVersionKind"); + }); - Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature); - } - } else { - assert(0 && "Expected a target or target_clones multiversion function"); + if (!ShouldEmitResolver) continue; - } llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); - if (FD->isTargetClonesMultiVersion()) { - const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + if (FD->isTargetClonesMultiVersion() && + !getTarget().getTriple().isAArch64()) { std::string MangledName = getMangledNameImpl( *this, GD, FD, /*OmitMultiVersionMangling=*/true); - // In prior versions of Clang, the mangling for ifuncs incorrectly - // included an .ifunc suffix. This alias is generated for backward - // compatibility. It is deprecated, and may be removed in the future. - auto *Alias = llvm::GlobalAlias::create( - DeclTy, 0, getMultiversionLinkage(*this, GD), - MangledName + ".ifunc", IFunc, &getModule()); - SetCommonAttributes(FD, Alias); + if (!GetGlobalValue(MangledName + ".ifunc")) { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + // In prior versions of Clang, the mangling for ifuncs incorrectly + // included an .ifunc suffix. This alias is generated for backward + // compatibility. It is deprecated, and may be removed in the future. + auto *Alias = llvm::GlobalAlias::create( + DeclTy, 0, getMultiversionLinkage(*this, GD), + MangledName + ".ifunc", IFunc, &getModule()); + SetCommonAttributes(FD, Alias); + } } } llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant); @@ -4276,6 +4317,14 @@ void CodeGenModule::emitMultiVersionFunctions() { emitMultiVersionFunctions(); } +static void replaceDeclarationWith(llvm::GlobalValue *Old, + llvm::Constant *New) { + assert(cast<llvm::Function>(Old)->isDeclaration() && "Not a declaration"); + New->takeName(Old); + Old->replaceAllUsesWith(New); + Old->eraseFromParent(); +} + void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Not a FunctionDecl?"); @@ -4380,12 +4429,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { // Fix up function declarations that were created for cpu_specific before // cpu_dispatch was known if (!isa<llvm::GlobalIFunc>(IFunc)) { - assert(cast<llvm::Function>(IFunc)->isDeclaration()); auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc, &getModule()); - GI->takeName(IFunc); - IFunc->replaceAllUsesWith(GI); - IFunc->eraseFromParent(); + replaceDeclarationWith(IFunc, GI); IFunc = GI; } @@ -4400,8 +4446,23 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { } } +/// Adds a declaration to the list of multi version functions if not present. +void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + assert(FD && "Not a FunctionDecl?"); + + if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) { + std::string MangledName = + getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true); + if (!DeferredResolversToEmit.insert(MangledName).second) + return; + } + MultiVersionFuncs.push_back(GD); +} + /// If a dispatcher for the specified mangled name is not in the module, create -/// and return an llvm Function with the specified type. +/// and return it. The dispatcher is either an llvm Function with the specified +/// type, or a global ifunc. llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Not a FunctionDecl?"); @@ -4413,14 +4474,31 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { // a separate resolver). std::string ResolverName = MangledName; if (getTarget().supportsIFunc()) { - if (!FD->isTargetClonesMultiVersion()) + switch (FD->getMultiVersionKind()) { + case MultiVersionKind::None: + llvm_unreachable("unexpected MultiVersionKind::None for resolver"); + case MultiVersionKind::Target: + case MultiVersionKind::CPUSpecific: + case MultiVersionKind::CPUDispatch: ResolverName += ".ifunc"; + break; + case MultiVersionKind::TargetClones: + case MultiVersionKind::TargetVersion: + break; + } } else if (FD->isTargetMultiVersion()) { ResolverName += ".resolver"; } - // If the resolver has already been created, just return it. - if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName)) + // If the resolver has already been created, just return it. This lookup may + // yield a function declaration instead of a resolver on AArch64. That is + // because we didn't know whether a resolver will be generated when we first + // encountered a use of the symbol named after this resolver. Therefore, + // targets which support ifuncs should not return here unless we actually + // found an ifunc. + llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName); + if (ResolverGV && + (isa<llvm::GlobalIFunc>(ResolverGV) || !getTarget().supportsIFunc())) return ResolverGV; const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); @@ -4429,7 +4507,7 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { // The resolver needs to be created. For target and target_clones, defer // creation until the end of the TU. if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion()) - MultiVersionFuncs.push_back(GD); + AddDeferredMultiVersionResolverToEmit(GD); // For cpu_specific, don't create an ifunc yet because we don't know if the // cpu_dispatch will be emitted in this translation unit. @@ -4446,7 +4524,8 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { "", Resolver, &getModule()); GIF->setName(ResolverName); SetCommonAttributes(FD, GIF); - + if (ResolverGV) + replaceDeclarationWith(ResolverGV, GIF); return GIF; } @@ -4455,9 +4534,24 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { assert(isa<llvm::GlobalValue>(Resolver) && "Resolver should be created for the first time"); SetCommonAttributes(FD, cast<llvm::GlobalValue>(Resolver)); + if (ResolverGV) + replaceDeclarationWith(ResolverGV, Resolver); return Resolver; } +bool CodeGenModule::shouldDropDLLAttribute(const Decl *D, + const llvm::GlobalValue *GV) const { + auto SC = GV->getDLLStorageClass(); + if (SC == llvm::GlobalValue::DefaultStorageClass) + return false; + const Decl *MRD = D->getMostRecentDecl(); + return (((SC == llvm::GlobalValue::DLLImportStorageClass && + !MRD->hasAttr<DLLImportAttr>()) || + (SC == llvm::GlobalValue::DLLExportStorageClass && + !MRD->hasAttr<DLLExportAttr>())) && + !shouldMapVisibilityToDLLExport(cast<NamedDecl>(MRD))); +} + /// GetOrCreateLLVMFunction - If the specified mangled name is not in the /// module, create and return an llvm Function with the specified type. If there /// is something in the module with the specified name, return it potentially @@ -4471,6 +4565,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( ForDefinition_t IsForDefinition) { const Decl *D = GD.getDecl(); + std::string NameWithoutMultiVersionMangling; // Any attempts to use a MultiVersion function should result in retrieving // the iFunc instead. Name Mangling will handle the rest of the changes. if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) { @@ -4492,11 +4587,24 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) - return GetOrCreateMultiVersionResolver(GD); + if (!IsForDefinition) { + // On AArch64 we do not immediatelly emit an ifunc resolver when a + // function is used. Instead we defer the emission until we see a + // default definition. In the meantime we just reference the symbol + // without FMV mangling (it may or may not be replaced later). + if (getTarget().getTriple().isAArch64()) { + AddDeferredMultiVersionResolverToEmit(GD); + NameWithoutMultiVersionMangling = getMangledNameImpl( + *this, GD, FD, /*OmitMultiVersionMangling=*/true); + } else + return GetOrCreateMultiVersionResolver(GD); + } } } + if (!NameWithoutMultiVersionMangling.empty()) + MangledName = NameWithoutMultiVersionMangling; + // Lookup the entry, lazily creating it if necessary. llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (Entry) { @@ -4507,8 +4615,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && - !shouldMapVisibilityToDLLExport(cast_or_null<NamedDecl>(D))) { + if (D && shouldDropDLLAttribute(D, Entry)) { Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); setDSOLocal(Entry); } @@ -4767,6 +4874,10 @@ CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name, } } setDSOLocal(F); + // FIXME: We should use CodeGenModule::SetLLVMFunctionAttributes() instead + // of trying to approximate the attributes using the LLVM function + // signature. This requires revising the API of CreateRuntimeFunction(). + markRegisterParameterAttributes(F); } } @@ -4798,8 +4909,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && - !shouldMapVisibilityToDLLExport(D)) + if (D && shouldDropDLLAttribute(D, Entry)) Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D) @@ -5104,8 +5214,11 @@ void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) { EmitGlobalVarDefinition(D); } -void CodeGenModule::EmitExternalDeclaration(const VarDecl *D) { - EmitExternalVarDeclaration(D); +void CodeGenModule::EmitExternalDeclaration(const DeclaratorDecl *D) { + if (auto const *V = dyn_cast<const VarDecl>(D)) + EmitExternalVarDeclaration(V); + if (auto const *FD = dyn_cast<const FunctionDecl>(D)) + EmitExternalFunctionDeclaration(FD); } CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { @@ -5289,6 +5402,18 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, !IsDefinitionAvailableExternally && D->needsDestruction(getContext()) == QualType::DK_cxx_destructor; + // It is helpless to emit the definition for an available_externally variable + // which can't be marked as const. + // We don't need to check if it needs global ctor or dtor. See the above + // comment for ideas. + if (IsDefinitionAvailableExternally && + (!D->hasConstantInitialization() || + // TODO: Update this when we have interface to check constexpr + // destructor. + D->needsDestruction(getContext()) || + !D->getType().isConstantStorage(getContext(), true, true))) + return; + const VarDecl *InitDecl; const Expr *InitExpr = D->getAnyInitializer(InitDecl); @@ -5529,6 +5654,18 @@ void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) { } } +void CodeGenModule::EmitExternalFunctionDeclaration(const FunctionDecl *FD) { + if (CGDebugInfo *DI = getModuleDebugInfo()) + if (getCodeGenOpts().hasReducedDebugInfo()) { + auto *Ty = getTypes().ConvertType(FD->getType()); + StringRef MangledName = getMangledName(FD); + auto *Fn = dyn_cast<llvm::Function>( + GetOrCreateLLVMFunction(MangledName, Ty, FD, /* ForVTable */ false)); + if (!Fn->getSubprogram()) + DI->EmitFunctionDecl(FD, FD->getLocation(), FD->getType(), Fn); + } +} + static bool isVarDeclStrongDefinition(const ASTContext &Context, CodeGenModule &CGM, const VarDecl *D, bool NoCommon) { @@ -5688,15 +5825,17 @@ CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) { static void replaceUsesOfNonProtoConstant(llvm::Constant *old, llvm::Function *newFn) { // Fast path. - if (old->use_empty()) return; + if (old->use_empty()) + return; llvm::Type *newRetTy = newFn->getReturnType(); - SmallVector<llvm::Value*, 4> newArgs; + SmallVector<llvm::Value *, 4> newArgs; + + SmallVector<llvm::CallBase *> callSitesToBeRemovedFromParent; for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end(); - ui != ue; ) { - llvm::Value::use_iterator use = ui++; // Increment before the use is erased. - llvm::User *user = use->getUser(); + ui != ue; ui++) { + llvm::User *user = ui->getUser(); // Recognize and replace uses of bitcasts. Most calls to // unprototyped functions will use bitcasts. @@ -5708,8 +5847,9 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, // Recognize calls to the function. llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(user); - if (!callSite) continue; - if (!callSite->isCallee(&*use)) + if (!callSite) + continue; + if (!callSite->isCallee(&*ui)) continue; // If the return types don't match exactly, then we can't @@ -5778,6 +5918,10 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, if (callSite->getDebugLoc()) newCall->setDebugLoc(callSite->getDebugLoc()); + callSitesToBeRemovedFromParent.push_back(callSite); + } + + for (auto *callSite : callSitesToBeRemovedFromParent) { callSite->eraseFromParent(); } } @@ -5801,7 +5945,8 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) { auto DK = VD->isThisDeclarationADefinition(); - if (DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>()) + if ((DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>()) || + (LangOpts.CUDA && !shouldEmitCUDAGlobalVar(VD))) return; TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind(); @@ -5973,11 +6118,14 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { Aliases.push_back(GD); - llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType()); - llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(DeclTy); + // The resolver might not be visited yet. Specify a dummy non-function type to + // indicate IsIncompleteFunction. Either the type is ignored (if the resolver + // was emitted) or the whole function will be replaced (if the resolver has + // not been emitted). llvm::Constant *Resolver = - GetOrCreateLLVMFunction(IFA->getResolver(), ResolverTy, {}, + GetOrCreateLLVMFunction(IFA->getResolver(), VoidTy, {}, /*ForVTable=*/false); + llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType()); llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create(DeclTy, 0, llvm::Function::ExternalLinkage, "", Resolver, &getModule()); @@ -6001,9 +6149,6 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { Entry->eraseFromParent(); } else GIF->setName(MangledName); - if (auto *F = dyn_cast<llvm::Function>(Resolver)) { - F->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation); - } SetCommonAttributes(GD, GIF); } @@ -6060,9 +6205,6 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { return ConstantAddress( C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment())); - llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); - llvm::Constant *Zeros[] = { Zero, Zero }; - const ASTContext &Context = getContext(); const llvm::Triple &Triple = getTriple(); @@ -6133,8 +6275,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { // Decay array -> ptr CFConstantStringClassRef = - IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) - : llvm::ConstantExpr::getGetElementPtr(Ty, C, Zeros); + IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) : C; } QualType CFTy = Context.getCFConstantStringType(); @@ -6190,10 +6331,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { GV->setSection(".rodata"); // String. - llvm::Constant *Str = - llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); - - Fields.add(Str); + Fields.add(GV); // String length. llvm::IntegerType *LengthTy = @@ -6287,7 +6425,7 @@ CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) { // Resize the string to the right size, which is indicated by its type. const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType()); assert(CAT && "String literal not of constant array type!"); - Str.resize(CAT->getSize().getZExtValue()); + Str.resize(CAT->getZExtSize()); return llvm::ConstantDataArray::getString(VMContext, Str, false); } @@ -6344,7 +6482,8 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, ConstantAddress CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, StringRef Name) { - CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(S->getType()); + CharUnits Alignment = + getContext().getAlignOfGlobalVarInChars(S->getType(), /*VD=*/nullptr); llvm::Constant *C = GetConstantArrayFromStringLiteral(S); llvm::GlobalVariable **Entry = nullptr; @@ -6407,8 +6546,8 @@ CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) { ConstantAddress CodeGenModule::GetAddrOfConstantCString( const std::string &Str, const char *GlobalName) { StringRef StrWithNull(Str.c_str(), Str.size() + 1); - CharUnits Alignment = - getContext().getAlignOfGlobalVarInChars(getContext().CharTy); + CharUnits Alignment = getContext().getAlignOfGlobalVarInChars( + getContext().CharTy, /*VD=*/nullptr); llvm::Constant *C = llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false); @@ -6623,7 +6762,7 @@ static bool AllTrivialInitializers(CodeGenModule &CGM, void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { // We might need a .cxx_destruct even if we don't have any ivar initializers. if (needsDestructMethod(D)) { - IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct"); + const IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct"); Selector cxxSelector = getContext().Selectors.getSelector(0, &II); ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create( getContext(), D->getLocation(), D->getLocation(), cxxSelector, @@ -6643,7 +6782,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { AllTrivialInitializers(*this, D)) return; - IdentifierInfo *II = &getContext().Idents.get(".cxx_construct"); + const IdentifierInfo *II = &getContext().Idents.get(".cxx_construct"); Selector cxxSelector = getContext().Selectors.getSelector(0, &II); // The constructor returns 'self'. ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create( @@ -7041,6 +7180,9 @@ void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) { SourceManager &SM = getContext().getSourceManager(); if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(D->getBeginLoc())) break; + if (!llvm::coverage::SystemHeadersCoverage && + SM.isInSystemHeader(D->getBeginLoc())) + break; DeferredEmptyCoverageMappingDecls.try_emplace(D, true); break; } @@ -7211,7 +7353,7 @@ void CodeGenModule::EmitStaticExternCAliases() { if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases()) return; for (auto &I : StaticExternCValues) { - IdentifierInfo *Name = I.first; + const IdentifierInfo *Name = I.first; llvm::GlobalValue *Val = I.second; // If Val is null, that implies there were multiple declarations that each @@ -7272,7 +7414,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second.getPointer(); + llvm::Value *Addr = I.second.emitRawPointer(*this); if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( |